{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a2a2d785",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T03:27:54.856947Z",
     "iopub.status.busy": "2023-10-06T03:27:54.856582Z",
     "iopub.status.idle": "2023-10-06T03:27:59.844014Z",
     "shell.execute_reply": "2023-10-06T03:27:59.842882Z"
    },
    "papermill": {
     "duration": 4.994131,
     "end_time": "2023-10-06T03:27:59.846368",
     "exception": false,
     "start_time": "2023-10-06T03:27:54.852237",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import optuna\n",
    "from sklearn.decomposition import PCA\n",
    "from lightgbm import LGBMClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.pipeline import make_pipeline, Pipeline\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler, LabelEncoder\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier, HistGradientBoostingClassifier, GradientBoostingClassifier, ExtraTreesClassifier\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a9b641e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T03:27:59.854636Z",
     "iopub.status.busy": "2023-10-06T03:27:59.854229Z",
     "iopub.status.idle": "2023-10-06T03:28:00.647055Z",
     "shell.execute_reply": "2023-10-06T03:28:00.645691Z"
    },
    "papermill": {
     "duration": 0.801395,
     "end_time": "2023-10-06T03:28:00.651208",
     "exception": false,
     "start_time": "2023-10-06T03:27:59.849813",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((112648, 22), (101763, 22), (67842, 21))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('/kaggle/input/playground-series-s3e23/train.csv').drop('id',axis=1)\n",
    "test = pd.read_csv('/kaggle/input/playground-series-s3e23/test.csv').drop('id',axis=1)\n",
    "origin = pd.read_csv('/kaggle/input/software-defect-prediction/jm1.csv')\n",
    "origin['defects'] = origin['defects'].map({False: 0, True: 1})\n",
    "train_total = pd.concat([train, origin], axis=0, ignore_index=True)\n",
    "sample_submission = pd.read_csv('/kaggle/input/playground-series-s3e23/sample_submission.csv')\n",
    "train_total.shape, train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2ca8b3ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T03:28:00.664254Z",
     "iopub.status.busy": "2023-10-06T03:28:00.663886Z",
     "iopub.status.idle": "2023-10-06T03:28:00.677772Z",
     "shell.execute_reply": "2023-10-06T03:28:00.676782Z"
    },
    "papermill": {
     "duration": 0.021242,
     "end_time": "2023-10-06T03:28:00.679851",
     "exception": false,
     "start_time": "2023-10-06T03:28:00.658609",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X = train.drop(columns = ['defects'], axis = 1)\n",
    "Y = train['defects']\n",
    "\n",
    "test_cv = test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "79037f22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T03:28:00.689001Z",
     "iopub.status.busy": "2023-10-06T03:28:00.688123Z",
     "iopub.status.idle": "2023-10-06T03:28:00.702463Z",
     "shell.execute_reply": "2023-10-06T03:28:00.701404Z"
    },
    "papermill": {
     "duration": 0.022425,
     "end_time": "2023-10-06T03:28:00.705636",
     "exception": false,
     "start_time": "2023-10-06T03:28:00.683211",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def hill_climbing(x, y, x_test):\n",
    "    # 初始化得分字典，用于存储每个模型的AUC得分\n",
    "    scores = {}\n",
    "\n",
    "    # 遍历x列标签（各模型）\n",
    "    for col in x.columns:\n",
    "        # 计算各模型的AUC指标\n",
    "        scores[col] = roc_auc_score(y, x[col])\n",
    "\n",
    "    # 根据AUC对模型进行排序，得分高的排在前面\n",
    "    scores = {k: v for k, v in sorted(scores.items(), key=lambda item: item[1], reverse=True)}\n",
    "\n",
    "    # 根据AUC得分高低重新排列x和x_test的列顺序\n",
    "    x = x[list(scores.keys())]\n",
    "    x_test = x_test[list(scores.keys())]\n",
    "\n",
    "    # 停止标志，控制爬山算法的终止条件\n",
    "    STOP = False\n",
    "\n",
    "    # 取出得分最高的模型作为初始最优模型\n",
    "    current_best_ensemble = x.iloc[:, 0]\n",
    "    current_best_test_preds = x_test.iloc[:, 0]\n",
    "\n",
    "    # 除最优模型外的其他模型\n",
    "    MODELS = x.iloc[:, 1:]\n",
    "\n",
    "    # 权重空间，用于尝试不同的权重值\n",
    "    weight_range = np.arange(-0.5, 0.51, 0.01)\n",
    "\n",
    "    # 计算最优模型AUC得分的历史，用于观察算法进展\n",
    "    history = [roc_auc_score(y, current_best_ensemble)]\n",
    "    j = 0\n",
    "\n",
    "    while not STOP:\n",
    "        j += 1\n",
    "        potential_new_best_cv_score = roc_auc_score(y, current_best_ensemble)\n",
    "        k_best, wgt_best = None, None\n",
    "\n",
    "        # 遍历每个模型以及权重空间，寻找最优的组合\n",
    "        for k in MODELS:\n",
    "            for wgt in weight_range:\n",
    "                potential_ensemble = (1 - wgt) * current_best_ensemble + wgt * MODELS[k]\n",
    "                cv_score = roc_auc_score(y, potential_ensemble)\n",
    "\n",
    "                # 如果当前组合的AUC得分高于历史最优，更新最优组合\n",
    "                if cv_score > potential_new_best_cv_score:\n",
    "                    potential_new_best_cv_score = cv_score\n",
    "                    k_best, wgt_best = k, wgt\n",
    "\n",
    "        if k_best is not None:\n",
    "            # 更新最优模型和测试集预测\n",
    "            current_best_ensemble = (1 - wgt_best) * current_best_ensemble + wgt_best * MODELS[k_best]\n",
    "            current_best_test_preds = (1 - wgt_best) * current_best_test_preds + wgt_best * x_test[k_best]\n",
    "\n",
    "            # 从可用模型中移除已选模型\n",
    "            MODELS.drop(k_best, axis=1, inplace=True)\n",
    "\n",
    "            # 如果已经没有其他可用模型，则停止\n",
    "            if MODELS.shape[1] == 0:\n",
    "                STOP = True\n",
    "\n",
    "            # 记录当前最优得分\n",
    "            history.append(potential_new_best_cv_score)\n",
    "        else:\n",
    "            # 如果无法找到更好的组合，则停止\n",
    "            STOP = True\n",
    "\n",
    "    # 返回最终得到的集成模型的预测结果\n",
    "    hill_ens_pred_1 = current_best_ensemble\n",
    "    hill_ens_pred_2 = current_best_test_preds\n",
    "\n",
    "    return [hill_ens_pred_1, hill_ens_pred_2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4a071ecf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T03:28:00.717633Z",
     "iopub.status.busy": "2023-10-06T03:28:00.717292Z",
     "iopub.status.idle": "2023-10-06T03:28:00.748124Z",
     "shell.execute_reply": "2023-10-06T03:28:00.746906Z"
    },
    "papermill": {
     "duration": 0.040136,
     "end_time": "2023-10-06T03:28:00.750788",
     "exception": false,
     "start_time": "2023-10-06T03:28:00.710652",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def rf_search(trial):\n",
    "    n_estimators = trial.suggest_int(\"n_estimators\", 200, 1000, step=50)\n",
    "    max_depth = trial.suggest_int(\"max_depth\", 5, 100, step=2)\n",
    "    min_samples_split = trial.suggest_int(\"min_samples_split\", 5, 100, step=2)\n",
    "    min_samples_leaf = trial.suggest_int(\"min_samples_leaf\", 5, 100, step=2)\n",
    "    sk = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 1, random_state = 42)\n",
    "    rf = RandomForestClassifier(n_estimators=n_estimators,\n",
    "                                min_samples_split=min_samples_split,\n",
    "                                min_samples_leaf = min_samples_leaf,\n",
    "                                max_depth=max_depth)\n",
    "    val = cross_val_score(rf, X, Y, scoring='roc_auc', cv=sk).mean()\n",
    "    return -val\n",
    "\n",
    "def et_search(trial):\n",
    "    n_estimators = trial.suggest_int(\"n_estimators\", 200, 1000, step=50)\n",
    "    max_depth = trial.suggest_int(\"max_depth\", 5, 100, step=2)\n",
    "    min_samples_split = trial.suggest_int(\"min_samples_split\", 5, 100, step=2)\n",
    "    min_samples_leaf = trial.suggest_int(\"min_samples_leaf\", 5, 100, step=2)\n",
    "    sk = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 1, random_state = 42)\n",
    "    et = ExtraTreesClassifier(n_estimators = n_estimators, \n",
    "                                 max_depth = max_depth,\n",
    "                                 min_samples_split = min_samples_split,\n",
    "                                 min_samples_leaf = min_samples_leaf)\n",
    "    val = cross_val_score(et, X, Y, scoring='roc_auc', cv=sk).mean()\n",
    "    return -val\n",
    "\n",
    "def hist_search(trial):\n",
    "    l2_regularization = trial.suggest_float('l2_regularization', 0.001, 0.1, log=True)\n",
    "    learning_rate = trial.suggest_float('learning_rate', 0.001, 0.1, log=True)\n",
    "    max_iter = trial.suggest_int('max_iter', 200, 1000, step=50)\n",
    "    max_depth = trial.suggest_int('max_depth', 5, 100, step=2)\n",
    "    min_samples_leaf = trial.suggest_int('min_samples_leaf', 10, 100, step=2)\n",
    "    max_leaf_nodes = trial.suggest_int('max_leaf_nodes', 5, 100, step=2)\n",
    "    sk = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 1, random_state = 42)\n",
    "    hist = HistGradientBoostingClassifier(l2_regularization = l2_regularization,\n",
    "                                          early_stopping = False,\n",
    "                                          learning_rate = learning_rate,\n",
    "                                          max_iter = max_iter,\n",
    "                                          max_depth = max_depth,\n",
    "                                          max_bins = 255,\n",
    "                                          min_samples_leaf = min_samples_leaf,\n",
    "                                          max_leaf_nodes = max_leaf_nodes)\n",
    "    val = cross_val_score(hist, X, Y, scoring='roc_auc', cv=sk).mean()\n",
    "    return -val\n",
    "\n",
    "def xgb_search(trial):\n",
    "    colsample_bytree = trial.suggest_float('colsample_bytree', 0.6, 1.0, step=0.1)\n",
    "    gamma = trial.suggest_float('gamma', 1, 10, step=0.5)\n",
    "    learning_rate = trial.suggest_float('learning_rate', 0.001, 0.1, log=True)\n",
    "    max_depth = trial.suggest_int('max_depth', 5, 100, step=2)\n",
    "    min_child_weight = trial.suggest_int('min_child_weight', 1, 15, step=1)\n",
    "    n_estimators = trial.suggest_int('n_estimators', 200, 1000, step=50)\n",
    "    subsample = trial.suggest_float('subsample', 0.6, 1.0, step=0.1)\n",
    "    sk = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 1, random_state = 42)\n",
    "    xgb = XGBClassifier(objective = 'binary:logistic',\n",
    "                           tree_method = 'hist',\n",
    "                           colsample_bytree = colsample_bytree, \n",
    "                           gamma = gamma, \n",
    "                           learning_rate = learning_rate, \n",
    "                           max_depth = max_depth, \n",
    "                           min_child_weight = min_child_weight, \n",
    "                           n_estimators = n_estimators, \n",
    "                           subsample = subsample)\n",
    "    val = cross_val_score(xgb, X, Y, scoring='roc_auc', cv=sk).mean()\n",
    "    return -val\n",
    "\n",
    "def cat_search(trial):\n",
    "    iterations = trial.suggest_int('iterations', 200, 1000, step=50)\n",
    "    learning_rate = trial.suggest_float('learning_rate', 0.001, 0.1, log=True)\n",
    "    depth = trial.suggest_int('depth', 5, 16, step=2)\n",
    "    random_strength = trial.suggest_float('random_strength', 0.5, 1, step=0.1)\n",
    "    bagging_temperature = trial.suggest_float('bagging_temperature', 0.5, 1, step=0.1)\n",
    "    border_count = trial.suggest_int('border_count', 1, 100, step=10)\n",
    "    l2_leaf_reg = trial.suggest_int('l2_leaf_reg', 1, 10, step=1)\n",
    "    sk = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 1, random_state = 42)\n",
    "    cat = CatBoostClassifier(loss_function = 'Logloss',\n",
    "                                iterations = iterations,\n",
    "                                learning_rate = learning_rate,\n",
    "                                depth = depth,\n",
    "                                random_strength = random_strength,\n",
    "                                bagging_temperature = bagging_temperature,\n",
    "                                border_count = border_count,\n",
    "                                l2_leaf_reg = l2_leaf_reg,\n",
    "                                verbose = False, \n",
    "                                task_type = 'CPU')\n",
    "    val = cross_val_score(cat, X, Y, scoring='roc_auc', cv=sk).mean()\n",
    "    return -val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "26350e8b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T03:28:00.763246Z",
     "iopub.status.busy": "2023-10-06T03:28:00.762873Z",
     "iopub.status.idle": "2023-10-06T14:01:53.992435Z",
     "shell.execute_reply": "2023-10-06T14:01:53.991283Z"
    },
    "papermill": {
     "duration": 38033.262426,
     "end_time": "2023-10-06T14:01:54.018303",
     "exception": false,
     "start_time": "2023-10-06T03:28:00.755877",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2023-10-06 03:28:00,766] A new study created in memory with name: rf\n",
      "[I 2023-10-06 03:34:35,235] Trial 0 finished with value: -0.7900455385600692 and parameters: {'n_estimators': 250, 'max_depth': 19, 'min_samples_split': 99, 'min_samples_leaf': 45}. Best is trial 0 with value: -0.7900455385600692.\n",
      "[I 2023-10-06 04:04:47,573] Trial 1 finished with value: -0.7896404897373415 and parameters: {'n_estimators': 1000, 'max_depth': 73, 'min_samples_split': 89, 'min_samples_leaf': 15}. Best is trial 0 with value: -0.7900455385600692.\n",
      "[I 2023-10-06 04:13:39,036] Trial 2 finished with value: -0.7893346613663202 and parameters: {'n_estimators': 300, 'max_depth': 57, 'min_samples_split': 71, 'min_samples_leaf': 25}. Best is trial 0 with value: -0.7900455385600692.\n",
      "[I 2023-10-06 04:31:18,432] Trial 3 finished with value: -0.7898661237037052 and parameters: {'n_estimators': 950, 'max_depth': 9, 'min_samples_split': 91, 'min_samples_leaf': 85}. Best is trial 0 with value: -0.7900455385600692.\n",
      "[I 2023-10-06 04:36:55,818] Trial 4 finished with value: -0.7896676210928564 and parameters: {'n_estimators': 200, 'max_depth': 59, 'min_samples_split': 57, 'min_samples_leaf': 41}. Best is trial 0 with value: -0.7900455385600692.\n",
      "[I 2023-10-06 04:44:25,606] Trial 5 finished with value: -0.789191184894716 and parameters: {'n_estimators': 250, 'max_depth': 85, 'min_samples_split': 55, 'min_samples_leaf': 27}. Best is trial 0 with value: -0.7900455385600692.\n",
      "[I 2023-10-06 05:09:43,657] Trial 6 finished with value: -0.7900626229106165 and parameters: {'n_estimators': 950, 'max_depth': 57, 'min_samples_split': 41, 'min_samples_leaf': 51}. Best is trial 6 with value: -0.7900626229106165.\n",
      "[I 2023-10-06 05:24:17,442] Trial 7 finished with value: -0.7901660827418848 and parameters: {'n_estimators': 600, 'max_depth': 27, 'min_samples_split': 35, 'min_samples_leaf': 83}. Best is trial 7 with value: -0.7901660827418848.\n",
      "[I 2023-10-06 05:31:28,130] Trial 8 finished with value: -0.7895992201544034 and parameters: {'n_estimators': 250, 'max_depth': 21, 'min_samples_split': 33, 'min_samples_leaf': 29}. Best is trial 7 with value: -0.7901660827418848.\n",
      "[I 2023-10-06 05:31:28,132] A new study created in memory with name: et\n",
      "[I 2023-10-06 05:34:00,131] Trial 0 finished with value: -0.7783705251555783 and parameters: {'n_estimators': 850, 'max_depth': 69, 'min_samples_split': 39, 'min_samples_leaf': 89}. Best is trial 0 with value: -0.7783705251555783.\n",
      "[I 2023-10-06 05:37:01,294] Trial 1 finished with value: -0.7789169750334339 and parameters: {'n_estimators': 950, 'max_depth': 63, 'min_samples_split': 13, 'min_samples_leaf': 73}. Best is trial 1 with value: -0.7789169750334339.\n",
      "[I 2023-10-06 05:37:44,574] Trial 2 finished with value: -0.7767859284934155 and parameters: {'n_estimators': 250, 'max_depth': 11, 'min_samples_split': 33, 'min_samples_leaf': 65}. Best is trial 1 with value: -0.7789169750334339.\n",
      "[I 2023-10-06 05:44:11,389] Trial 3 finished with value: -0.78592179365384 and parameters: {'n_estimators': 650, 'max_depth': 83, 'min_samples_split': 79, 'min_samples_leaf': 7}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 05:45:52,381] Trial 4 finished with value: -0.7795560847435604 and parameters: {'n_estimators': 500, 'max_depth': 33, 'min_samples_split': 95, 'min_samples_leaf': 67}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 05:48:12,439] Trial 5 finished with value: -0.7808217821439436 and parameters: {'n_estimators': 600, 'max_depth': 31, 'min_samples_split': 33, 'min_samples_leaf': 45}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 05:50:49,250] Trial 6 finished with value: -0.779181470718226 and parameters: {'n_estimators': 800, 'max_depth': 67, 'min_samples_split': 11, 'min_samples_leaf': 69}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 05:53:52,811] Trial 7 finished with value: -0.780113766321648 and parameters: {'n_estimators': 850, 'max_depth': 35, 'min_samples_split': 67, 'min_samples_leaf': 55}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 05:56:40,055] Trial 8 finished with value: -0.7782943250724277 and parameters: {'n_estimators': 900, 'max_depth': 17, 'min_samples_split': 85, 'min_samples_leaf': 71}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 05:59:09,413] Trial 9 finished with value: -0.7832319934747466 and parameters: {'n_estimators': 450, 'max_depth': 85, 'min_samples_split': 5, 'min_samples_leaf': 23}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 06:00:56,304] Trial 10 finished with value: -0.7850977155913751 and parameters: {'n_estimators': 200, 'max_depth': 97, 'min_samples_split': 69, 'min_samples_leaf': 9}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 06:02:41,259] Trial 11 finished with value: -0.7853221726519456 and parameters: {'n_estimators': 200, 'max_depth': 95, 'min_samples_split': 67, 'min_samples_leaf': 9}. Best is trial 3 with value: -0.78592179365384.\n",
      "[I 2023-10-06 06:10:34,686] Trial 12 finished with value: -0.7861841164994876 and parameters: {'n_estimators': 700, 'max_depth': 95, 'min_samples_split': 69, 'min_samples_leaf': 5}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:13:49,391] Trial 13 finished with value: -0.7823778104260859 and parameters: {'n_estimators': 700, 'max_depth': 81, 'min_samples_split': 81, 'min_samples_leaf': 31}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:17:27,333] Trial 14 finished with value: -0.7827580254918935 and parameters: {'n_estimators': 700, 'max_depth': 81, 'min_samples_split': 51, 'min_samples_leaf': 25}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:22:03,277] Trial 15 finished with value: -0.785993400242516 and parameters: {'n_estimators': 400, 'max_depth': 51, 'min_samples_split': 55, 'min_samples_leaf': 5}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:23:38,022] Trial 16 finished with value: -0.7819005125682995 and parameters: {'n_estimators': 350, 'max_depth': 49, 'min_samples_split': 55, 'min_samples_leaf': 33}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:27:17,413] Trial 17 finished with value: -0.7844258256075246 and parameters: {'n_estimators': 500, 'max_depth': 51, 'min_samples_split': 51, 'min_samples_leaf': 15}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:28:41,112] Trial 18 finished with value: -0.7808189045843538 and parameters: {'n_estimators': 350, 'max_depth': 43, 'min_samples_split': 59, 'min_samples_leaf': 45}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:29:44,489] Trial 19 finished with value: -0.7779269427742118 and parameters: {'n_estimators': 350, 'max_depth': 59, 'min_samples_split': 95, 'min_samples_leaf': 97}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:34:13,040] Trial 20 finished with value: -0.7833464075767331 and parameters: {'n_estimators': 750, 'max_depth': 23, 'min_samples_split': 43, 'min_samples_leaf': 19}. Best is trial 12 with value: -0.7861841164994876.\n",
      "[I 2023-10-06 06:41:11,255] Trial 21 finished with value: -0.7862751728139099 and parameters: {'n_estimators': 600, 'max_depth': 73, 'min_samples_split': 77, 'min_samples_leaf': 5}. Best is trial 21 with value: -0.7862751728139099.\n",
      "[I 2023-10-06 06:47:31,451] Trial 22 finished with value: -0.7862825305349352 and parameters: {'n_estimators': 550, 'max_depth': 73, 'min_samples_split': 75, 'min_samples_leaf': 5}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 06:51:31,463] Trial 23 finished with value: -0.7844632592452598 and parameters: {'n_estimators': 550, 'max_depth': 73, 'min_samples_split': 75, 'min_samples_leaf': 15}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 06:54:16,034] Trial 24 finished with value: -0.7819733623499661 and parameters: {'n_estimators': 600, 'max_depth': 91, 'min_samples_split': 87, 'min_samples_leaf': 33}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 06:58:54,544] Trial 25 finished with value: -0.7842826529237367 and parameters: {'n_estimators': 650, 'max_depth': 75, 'min_samples_split': 71, 'min_samples_leaf': 15}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 07:04:00,765] Trial 26 finished with value: -0.7825101034418581 and parameters: {'n_estimators': 1000, 'max_depth': 59, 'min_samples_split': 63, 'min_samples_leaf': 27}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 07:06:16,735] Trial 27 finished with value: -0.7813763708719569 and parameters: {'n_estimators': 550, 'max_depth': 89, 'min_samples_split': 89, 'min_samples_leaf': 41}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 07:10:50,866] Trial 28 finished with value: -0.7837975612873839 and parameters: {'n_estimators': 750, 'max_depth': 99, 'min_samples_split': 75, 'min_samples_leaf': 19}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 07:20:21,163] Trial 29 finished with value: -0.7860714338920223 and parameters: {'n_estimators': 800, 'max_depth': 75, 'min_samples_split': 45, 'min_samples_leaf': 5}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 07:22:24,860] Trial 30 finished with value: -0.7781599212993348 and parameters: {'n_estimators': 650, 'max_depth': 69, 'min_samples_split': 81, 'min_samples_leaf': 83}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 07:32:32,012] Trial 31 finished with value: -0.7860412842712632 and parameters: {'n_estimators': 850, 'max_depth': 77, 'min_samples_split': 43, 'min_samples_leaf': 5}. Best is trial 22 with value: -0.7862825305349352.\n",
      "[I 2023-10-06 07:32:32,014] A new study created in memory with name: hist\n",
      "[I 2023-10-06 07:33:03,369] Trial 0 finished with value: -0.7912048832332232 and parameters: {'l2_regularization': 0.0011090130243162505, 'learning_rate': 0.06657967933151847, 'max_iter': 350, 'max_depth': 13, 'min_samples_leaf': 14, 'max_leaf_nodes': 19}. Best is trial 0 with value: -0.7912048832332232.\n",
      "[I 2023-10-06 07:36:51,403] Trial 1 finished with value: -0.7908400877443342 and parameters: {'l2_regularization': 0.05848638392931617, 'learning_rate': 0.0015021394261892794, 'max_iter': 900, 'max_depth': 27, 'min_samples_leaf': 28, 'max_leaf_nodes': 87}. Best is trial 0 with value: -0.7912048832332232.\n",
      "[I 2023-10-06 07:39:01,086] Trial 2 finished with value: -0.7920988456786131 and parameters: {'l2_regularization': 0.01819127299754117, 'learning_rate': 0.005261225037785109, 'max_iter': 800, 'max_depth': 17, 'min_samples_leaf': 16, 'max_leaf_nodes': 43}. Best is trial 2 with value: -0.7920988456786131.\n",
      "[I 2023-10-06 07:39:54,500] Trial 3 finished with value: -0.7906904915845641 and parameters: {'l2_regularization': 0.06797579612438223, 'learning_rate': 0.005812024722245595, 'max_iter': 200, 'max_depth': 15, 'min_samples_leaf': 48, 'max_leaf_nodes': 85}. Best is trial 2 with value: -0.7920988456786131.\n",
      "[I 2023-10-06 07:42:25,381] Trial 4 finished with value: -0.7910878779557678 and parameters: {'l2_regularization': 0.07391225442753284, 'learning_rate': 0.011286441091521068, 'max_iter': 800, 'max_depth': 69, 'min_samples_leaf': 62, 'max_leaf_nodes': 69}. Best is trial 2 with value: -0.7920988456786131.\n",
      "[I 2023-10-06 07:43:28,702] Trial 5 finished with value: -0.7920628867661382 and parameters: {'l2_regularization': 0.013898919633049574, 'learning_rate': 0.016736045463890327, 'max_iter': 350, 'max_depth': 41, 'min_samples_leaf': 74, 'max_leaf_nodes': 55}. Best is trial 2 with value: -0.7920988456786131.\n",
      "[I 2023-10-06 07:46:26,033] Trial 6 finished with value: -0.7912830539976657 and parameters: {'l2_regularization': 0.00573282962588683, 'learning_rate': 0.008377841722303372, 'max_iter': 900, 'max_depth': 83, 'min_samples_leaf': 20, 'max_leaf_nodes': 75}. Best is trial 2 with value: -0.7920988456786131.\n",
      "[I 2023-10-06 07:47:37,159] Trial 7 finished with value: -0.7921145087240273 and parameters: {'l2_regularization': 0.025543816196830826, 'learning_rate': 0.013717490367468882, 'max_iter': 700, 'max_depth': 5, 'min_samples_leaf': 14, 'max_leaf_nodes': 57}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:49:29,463] Trial 8 finished with value: -0.7907407485158365 and parameters: {'l2_regularization': 0.020415384805225336, 'learning_rate': 0.0020540407840337017, 'max_iter': 550, 'max_depth': 25, 'min_samples_leaf': 18, 'max_leaf_nodes': 57}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:51:48,317] Trial 9 finished with value: -0.7905247953042447 and parameters: {'l2_regularization': 0.0023184986476668798, 'learning_rate': 0.013855402510013916, 'max_iter': 700, 'max_depth': 37, 'min_samples_leaf': 34, 'max_leaf_nodes': 79}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:52:49,620] Trial 10 finished with value: -0.7906711636081283 and parameters: {'l2_regularization': 0.006793266824520825, 'learning_rate': 0.0411231379430855, 'max_iter': 550, 'max_depth': 99, 'min_samples_leaf': 44, 'max_leaf_nodes': 31}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:54:18,363] Trial 11 finished with value: -0.7914257326934833 and parameters: {'l2_regularization': 0.02606472155142518, 'learning_rate': 0.0036472710080437953, 'max_iter': 700, 'max_depth': 5, 'min_samples_leaf': 98, 'max_leaf_nodes': 39}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:55:24,771] Trial 12 finished with value: -0.7902572518270758 and parameters: {'l2_regularization': 0.02822072159343958, 'learning_rate': 0.004334130378648877, 'max_iter': 1000, 'max_depth': 57, 'min_samples_leaf': 10, 'max_leaf_nodes': 5}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:56:30,117] Trial 13 finished with value: -0.7920463360274522 and parameters: {'l2_regularization': 0.011001161019911642, 'learning_rate': 0.021385369749784243, 'max_iter': 700, 'max_depth': 5, 'min_samples_leaf': 32, 'max_leaf_nodes': 39}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:58:40,672] Trial 14 finished with value: -0.7880719707122374 and parameters: {'l2_regularization': 0.03494634611398891, 'learning_rate': 0.03004314430672222, 'max_iter': 800, 'max_depth': 25, 'min_samples_leaf': 58, 'max_leaf_nodes': 63}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 07:59:55,692] Trial 15 finished with value: -0.7920043897951822 and parameters: {'l2_regularization': 0.017011750420409595, 'learning_rate': 0.007400879722003323, 'max_iter': 450, 'max_depth': 47, 'min_samples_leaf': 72, 'max_leaf_nodes': 43}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 08:03:29,205] Trial 16 finished with value: -0.7910834292554484 and parameters: {'l2_regularization': 0.04016127625478628, 'learning_rate': 0.0029519989755915827, 'max_iter': 800, 'max_depth': 59, 'min_samples_leaf': 40, 'max_leaf_nodes': 99}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 08:04:56,363] Trial 17 finished with value: -0.7891521112476593 and parameters: {'l2_regularization': 0.09580531283924995, 'learning_rate': 0.0010160161691802406, 'max_iter': 650, 'max_depth': 17, 'min_samples_leaf': 24, 'max_leaf_nodes': 25}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 08:07:25,695] Trial 18 finished with value: -0.791757126273645 and parameters: {'l2_regularization': 0.04429898708379775, 'learning_rate': 0.010467328924358525, 'max_iter': 1000, 'max_depth': 37, 'min_samples_leaf': 98, 'max_leaf_nodes': 49}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 08:08:45,903] Trial 19 finished with value: -0.7920000890671781 and parameters: {'l2_regularization': 0.017404694490057823, 'learning_rate': 0.0057453214035405875, 'max_iter': 900, 'max_depth': 5, 'min_samples_leaf': 12, 'max_leaf_nodes': 13}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 08:10:28,545] Trial 20 finished with value: -0.7908683952971023 and parameters: {'l2_regularization': 0.008101252489091592, 'learning_rate': 0.01947938622075323, 'max_iter': 600, 'max_depth': 31, 'min_samples_leaf': 84, 'max_leaf_nodes': 63}. Best is trial 7 with value: -0.7921145087240273.\n",
      "[I 2023-10-06 08:11:30,622] Trial 21 finished with value: -0.7922042156527302 and parameters: {'l2_regularization': 0.012786181749008604, 'learning_rate': 0.016018403258184998, 'max_iter': 350, 'max_depth': 43, 'min_samples_leaf': 72, 'max_leaf_nodes': 53}. Best is trial 21 with value: -0.7922042156527302.\n",
      "[I 2023-10-06 08:12:08,121] Trial 22 finished with value: -0.7916232372494981 and parameters: {'l2_regularization': 0.01180199734144468, 'learning_rate': 0.011568484046726262, 'max_iter': 200, 'max_depth': 17, 'min_samples_leaf': 68, 'max_leaf_nodes': 47}. Best is trial 21 with value: -0.7922042156527302.\n",
      "[I 2023-10-06 08:13:14,842] Trial 23 finished with value: -0.7920917161975317 and parameters: {'l2_regularization': 0.020561585950457418, 'learning_rate': 0.007318304502701462, 'max_iter': 450, 'max_depth': 69, 'min_samples_leaf': 84, 'max_leaf_nodes': 31}. Best is trial 21 with value: -0.7922042156527302.\n",
      "[I 2023-10-06 08:14:28,348] Trial 24 finished with value: -0.7908188967600186 and parameters: {'l2_regularization': 0.027194452494517903, 'learning_rate': 0.02530304900059045, 'max_iter': 450, 'max_depth': 45, 'min_samples_leaf': 50, 'max_leaf_nodes': 57}. Best is trial 21 with value: -0.7922042156527302.\n",
      "[I 2023-10-06 08:15:13,703] Trial 25 finished with value: -0.7924229497676007 and parameters: {'l2_regularization': 0.011280087211944019, 'learning_rate': 0.014810065932245119, 'max_iter': 300, 'max_depth': 21, 'min_samples_leaf': 80, 'max_leaf_nodes': 35}. Best is trial 25 with value: -0.7924229497676007.\n",
      "[I 2023-10-06 08:15:57,777] Trial 26 finished with value: -0.7924552855544078 and parameters: {'l2_regularization': 0.00969566596220893, 'learning_rate': 0.016041546673614744, 'max_iter': 300, 'max_depth': 53, 'min_samples_leaf': 84, 'max_leaf_nodes': 33}. Best is trial 26 with value: -0.7924552855544078.\n",
      "[I 2023-10-06 08:16:34,979] Trial 27 finished with value: -0.7922473154223402 and parameters: {'l2_regularization': 0.004945755280799658, 'learning_rate': 0.03191561719060124, 'max_iter': 300, 'max_depth': 55, 'min_samples_leaf': 84, 'max_leaf_nodes': 31}. Best is trial 26 with value: -0.7924552855544078.\n",
      "[I 2023-10-06 08:17:06,489] Trial 28 finished with value: -0.7924013160406242 and parameters: {'l2_regularization': 0.005100645962313793, 'learning_rate': 0.03502805663556399, 'max_iter': 250, 'max_depth': 65, 'min_samples_leaf': 88, 'max_leaf_nodes': 29}. Best is trial 26 with value: -0.7924552855544078.\n",
      "[I 2023-10-06 08:17:31,310] Trial 29 finished with value: -0.7916050064913442 and parameters: {'l2_regularization': 0.0043892746145635115, 'learning_rate': 0.07852957493556374, 'max_iter': 250, 'max_depth': 69, 'min_samples_leaf': 90, 'max_leaf_nodes': 21}. Best is trial 26 with value: -0.7924552855544078.\n",
      "[I 2023-10-06 08:17:56,632] Trial 30 finished with value: -0.7924696623495261 and parameters: {'l2_regularization': 0.008899810484868185, 'learning_rate': 0.04657150191483075, 'max_iter': 300, 'max_depth': 79, 'min_samples_leaf': 92, 'max_leaf_nodes': 15}. Best is trial 30 with value: -0.7924696623495261.\n",
      "[I 2023-10-06 08:18:23,015] Trial 31 finished with value: -0.7924072753544715 and parameters: {'l2_regularization': 0.008631616525180274, 'learning_rate': 0.0422857761346037, 'max_iter': 300, 'max_depth': 79, 'min_samples_leaf': 92, 'max_leaf_nodes': 13}. Best is trial 30 with value: -0.7924696623495261.\n",
      "[I 2023-10-06 08:18:56,070] Trial 32 finished with value: -0.7923250266660421 and parameters: {'l2_regularization': 0.009498896654594616, 'learning_rate': 0.04766752840717318, 'max_iter': 400, 'max_depth': 83, 'min_samples_leaf': 78, 'max_leaf_nodes': 13}. Best is trial 30 with value: -0.7924696623495261.\n",
      "[I 2023-10-06 08:19:22,502] Trial 33 finished with value: -0.7924622741144826 and parameters: {'l2_regularization': 0.009116646684798746, 'learning_rate': 0.051932115207693946, 'max_iter': 300, 'max_depth': 79, 'min_samples_leaf': 94, 'max_leaf_nodes': 13}. Best is trial 30 with value: -0.7924696623495261.\n",
      "[I 2023-10-06 08:19:50,932] Trial 34 finished with value: -0.79196164855395 and parameters: {'l2_regularization': 0.008827117196211083, 'learning_rate': 0.06771593025016795, 'max_iter': 300, 'max_depth': 95, 'min_samples_leaf': 100, 'max_leaf_nodes': 19}. Best is trial 30 with value: -0.7924696623495261.\n",
      "[I 2023-10-06 08:20:19,772] Trial 35 finished with value: -0.7924799471320163 and parameters: {'l2_regularization': 0.0067112560207155695, 'learning_rate': 0.05342731441557629, 'max_iter': 400, 'max_depth': 75, 'min_samples_leaf': 94, 'max_leaf_nodes': 9}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:20:45,932] Trial 36 finished with value: -0.7921712265144366 and parameters: {'l2_regularization': 0.0035151723053552506, 'learning_rate': 0.09502700526558075, 'max_iter': 400, 'max_depth': 75, 'min_samples_leaf': 94, 'max_leaf_nodes': 7}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:21:02,052] Trial 37 finished with value: -0.792285019747192 and parameters: {'l2_regularization': 0.006594965201222441, 'learning_rate': 0.05282387465320018, 'max_iter': 200, 'max_depth': 89, 'min_samples_leaf': 94, 'max_leaf_nodes': 9}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:21:43,166] Trial 38 finished with value: -0.7911176789249789 and parameters: {'l2_regularization': 0.007079651299217183, 'learning_rate': 0.056303608433305626, 'max_iter': 400, 'max_depth': 75, 'min_samples_leaf': 64, 'max_leaf_nodes': 25}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:22:25,501] Trial 39 finished with value: -0.7911132309322519 and parameters: {'l2_regularization': 0.014158805588244658, 'learning_rate': 0.06254600759675702, 'max_iter': 500, 'max_depth': 89, 'min_samples_leaf': 88, 'max_leaf_nodes': 17}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:22:50,205] Trial 40 finished with value: -0.7910899547089111 and parameters: {'l2_regularization': 0.003540027469642199, 'learning_rate': 0.09718799175064731, 'max_iter': 250, 'max_depth': 51, 'min_samples_leaf': 80, 'max_leaf_nodes': 21}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:23:37,523] Trial 41 finished with value: -0.792279960197749 and parameters: {'l2_regularization': 0.01082060956506772, 'learning_rate': 0.023328676040592234, 'max_iter': 350, 'max_depth': 61, 'min_samples_leaf': 82, 'max_leaf_nodes': 37}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:24:04,876] Trial 42 finished with value: -0.7924281359655967 and parameters: {'l2_regularization': 0.009790987442134727, 'learning_rate': 0.036950301657295, 'max_iter': 300, 'max_depth': 87, 'min_samples_leaf': 76, 'max_leaf_nodes': 15}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:24:33,105] Trial 43 finished with value: -0.792418827075422 and parameters: {'l2_regularization': 0.007932466141711716, 'learning_rate': 0.03986849074520106, 'max_iter': 350, 'max_depth': 91, 'min_samples_leaf': 76, 'max_leaf_nodes': 11}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:24:57,816] Trial 44 finished with value: -0.79243305719527 and parameters: {'l2_regularization': 0.005982858462089093, 'learning_rate': 0.04845483023351251, 'max_iter': 250, 'max_depth': 81, 'min_samples_leaf': 88, 'max_leaf_nodes': 17}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:25:21,237] Trial 45 finished with value: -0.7922307091069283 and parameters: {'l2_regularization': 0.0059545418418962374, 'learning_rate': 0.053384391690704865, 'max_iter': 200, 'max_depth': 75, 'min_samples_leaf': 96, 'max_leaf_nodes': 25}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:25:36,867] Trial 46 finished with value: -0.7922229505492097 and parameters: {'l2_regularization': 0.007066455802193038, 'learning_rate': 0.07295250738529116, 'max_iter': 250, 'max_depth': 81, 'min_samples_leaf': 88, 'max_leaf_nodes': 5}. Best is trial 35 with value: -0.7924799471320163.\n",
      "[I 2023-10-06 08:26:15,461] Trial 47 finished with value: -0.7925694708908781 and parameters: {'l2_regularization': 0.013929801807252486, 'learning_rate': 0.027652803974654264, 'max_iter': 400, 'max_depth': 65, 'min_samples_leaf': 100, 'max_leaf_nodes': 19}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:26:51,773] Trial 48 finished with value: -0.7923672214959627 and parameters: {'l2_regularization': 0.014619758486675937, 'learning_rate': 0.027895177246570705, 'max_iter': 500, 'max_depth': 65, 'min_samples_leaf': 100, 'max_leaf_nodes': 9}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:27:34,482] Trial 49 finished with value: -0.792340013182572 and parameters: {'l2_regularization': 0.014700136486622276, 'learning_rate': 0.029095727227009996, 'max_iter': 400, 'max_depth': 71, 'min_samples_leaf': 94, 'max_leaf_nodes': 23}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:28:31,860] Trial 50 finished with value: -0.7924505918716283 and parameters: {'l2_regularization': 0.021376700362606037, 'learning_rate': 0.020875147644318342, 'max_iter': 500, 'max_depth': 61, 'min_samples_leaf': 100, 'max_leaf_nodes': 27}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:29:30,303] Trial 51 finished with value: -0.7923026823587312 and parameters: {'l2_regularization': 0.019347166190274732, 'learning_rate': 0.01987716564609134, 'max_iter': 500, 'max_depth': 63, 'min_samples_leaf': 100, 'max_leaf_nodes': 29}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:30:24,439] Trial 52 finished with value: -0.7924313960414997 and parameters: {'l2_regularization': 0.010702886309576049, 'learning_rate': 0.023977904109706745, 'max_iter': 550, 'max_depth': 53, 'min_samples_leaf': 96, 'max_leaf_nodes': 19}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:30:59,929] Trial 53 finished with value: -0.7924793654721264 and parameters: {'l2_regularization': 0.012902861466939492, 'learning_rate': 0.03489091709650943, 'max_iter': 400, 'max_depth': 49, 'min_samples_leaf': 96, 'max_leaf_nodes': 15}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:31:29,713] Trial 54 finished with value: -0.7923866302659238 and parameters: {'l2_regularization': 0.016078201061215956, 'learning_rate': 0.035076823288375296, 'max_iter': 400, 'max_depth': 49, 'min_samples_leaf': 92, 'max_leaf_nodes': 9}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:31:52,522] Trial 55 finished with value: -0.7921378404924218 and parameters: {'l2_regularization': 0.013762993405380097, 'learning_rate': 0.043515020614738074, 'max_iter': 350, 'max_depth': 37, 'min_samples_leaf': 86, 'max_leaf_nodes': 5}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:32:31,546] Trial 56 finished with value: -0.7924815105478751 and parameters: {'l2_regularization': 0.012397413806260619, 'learning_rate': 0.03222155231935824, 'max_iter': 450, 'max_depth': 57, 'min_samples_leaf': 92, 'max_leaf_nodes': 15}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:33:10,923] Trial 57 finished with value: -0.7925160157473415 and parameters: {'l2_regularization': 0.01230391427337515, 'learning_rate': 0.03386039558785914, 'max_iter': 450, 'max_depth': 73, 'min_samples_leaf': 92, 'max_leaf_nodes': 15}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:34:01,442] Trial 58 finished with value: -0.7923905856396785 and parameters: {'l2_regularization': 0.012019977194346778, 'learning_rate': 0.03338893546388863, 'max_iter': 600, 'max_depth': 57, 'min_samples_leaf': 90, 'max_leaf_nodes': 15}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:34:47,850] Trial 59 finished with value: -0.7922832626388112 and parameters: {'l2_regularization': 0.017519573478251064, 'learning_rate': 0.027407875906655214, 'max_iter': 450, 'max_depth': 73, 'min_samples_leaf': 96, 'max_leaf_nodes': 21}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:36:16,787] Trial 60 finished with value: -0.7878544790188661 and parameters: {'l2_regularization': 0.011377750003913376, 'learning_rate': 0.03870327166268962, 'max_iter': 450, 'max_depth': 67, 'min_samples_leaf': 56, 'max_leaf_nodes': 83}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:36:44,535] Trial 61 finished with value: -0.7923734651880492 and parameters: {'l2_regularization': 0.012811572863279405, 'learning_rate': 0.05973508266367908, 'max_iter': 350, 'max_depth': 77, 'min_samples_leaf': 92, 'max_leaf_nodes': 11}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:37:20,105] Trial 62 finished with value: -0.792104296509649 and parameters: {'l2_regularization': 0.008403188041129304, 'learning_rate': 0.0471244340144425, 'max_iter': 400, 'max_depth': 83, 'min_samples_leaf': 96, 'max_leaf_nodes': 17}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:37:58,085] Trial 63 finished with value: -0.7924633591424117 and parameters: {'l2_regularization': 0.01306670039001239, 'learning_rate': 0.032201802390469526, 'max_iter': 450, 'max_depth': 71, 'min_samples_leaf': 92, 'max_leaf_nodes': 13}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:38:34,678] Trial 64 finished with value: -0.7923354708841431 and parameters: {'l2_regularization': 0.016017558488755097, 'learning_rate': 0.03117366525370798, 'max_iter': 550, 'max_depth': 71, 'min_samples_leaf': 90, 'max_leaf_nodes': 7}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:39:12,429] Trial 65 finished with value: -0.7923573276297391 and parameters: {'l2_regularization': 0.013231938032516821, 'learning_rate': 0.04286306319307282, 'max_iter': 450, 'max_depth': 57, 'min_samples_leaf': 98, 'max_leaf_nodes': 15}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:41:07,767] Trial 66 finished with value: -0.7904466347543863 and parameters: {'l2_regularization': 0.021907105666712962, 'learning_rate': 0.017762680821391437, 'max_iter': 600, 'max_depth': 67, 'min_samples_leaf': 82, 'max_leaf_nodes': 73}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:42:59,789] Trial 67 finished with value: -0.7893704030232211 and parameters: {'l2_regularization': 0.010421144467235686, 'learning_rate': 0.025447537255553734, 'max_iter': 500, 'max_depth': 61, 'min_samples_leaf': 86, 'max_leaf_nodes': 91}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:43:48,149] Trial 68 finished with value: -0.7920369033455652 and parameters: {'l2_regularization': 0.02366376707812535, 'learning_rate': 0.03168100137213922, 'max_iter': 450, 'max_depth': 71, 'min_samples_leaf': 92, 'max_leaf_nodes': 23}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:44:16,732] Trial 69 finished with value: -0.7923961162194147 and parameters: {'l2_regularization': 0.01817665643742848, 'learning_rate': 0.03841859901270853, 'max_iter': 350, 'max_depth': 85, 'min_samples_leaf': 70, 'max_leaf_nodes': 11}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:45:15,506] Trial 70 finished with value: -0.7916606448349796 and parameters: {'l2_regularization': 0.007504651354026823, 'learning_rate': 0.027285538283334576, 'max_iter': 400, 'max_depth': 41, 'min_samples_leaf': 98, 'max_leaf_nodes': 43}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:45:43,433] Trial 71 finished with value: -0.7923330675290072 and parameters: {'l2_regularization': 0.009209150602296233, 'learning_rate': 0.05071458907261349, 'max_iter': 350, 'max_depth': 77, 'min_samples_leaf': 94, 'max_leaf_nodes': 13}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:46:24,431] Trial 72 finished with value: -0.7914800914001151 and parameters: {'l2_regularization': 0.012670260928678498, 'learning_rate': 0.057454285995675224, 'max_iter': 450, 'max_depth': 79, 'min_samples_leaf': 90, 'max_leaf_nodes': 19}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:47:05,138] Trial 73 finished with value: -0.7923912853136059 and parameters: {'l2_regularization': 0.009599813033982408, 'learning_rate': 0.045041337280735574, 'max_iter': 650, 'max_depth': 65, 'min_samples_leaf': 86, 'max_leaf_nodes': 7}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:47:40,091] Trial 74 finished with value: -0.792502021000353 and parameters: {'l2_regularization': 0.00818343094883513, 'learning_rate': 0.03536980129149376, 'max_iter': 400, 'max_depth': 73, 'min_samples_leaf': 98, 'max_leaf_nodes': 15}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:48:17,135] Trial 75 finished with value: -0.7924079991653266 and parameters: {'l2_regularization': 0.015088877287313106, 'learning_rate': 0.03518348115422927, 'max_iter': 400, 'max_depth': 69, 'min_samples_leaf': 98, 'max_leaf_nodes': 17}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:49:12,364] Trial 76 finished with value: -0.7923974573750675 and parameters: {'l2_regularization': 0.00770608218438834, 'learning_rate': 0.022246381549780186, 'max_iter': 500, 'max_depth': 73, 'min_samples_leaf': 100, 'max_leaf_nodes': 23}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:49:44,754] Trial 77 finished with value: -0.7922121064387919 and parameters: {'l2_regularization': 0.011375989561310178, 'learning_rate': 0.030034931723878618, 'max_iter': 400, 'max_depth': 55, 'min_samples_leaf': 38, 'max_leaf_nodes': 11}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:50:23,120] Trial 78 finished with value: -0.7924044401341106 and parameters: {'l2_regularization': 0.0176991731989214, 'learning_rate': 0.037945069305275574, 'max_iter': 450, 'max_depth': 47, 'min_samples_leaf': 96, 'max_leaf_nodes': 15}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:51:02,112] Trial 79 finished with value: -0.7920539476847954 and parameters: {'l2_regularization': 0.008874009975885071, 'learning_rate': 0.042369042128976436, 'max_iter': 550, 'max_depth': 59, 'min_samples_leaf': 28, 'max_leaf_nodes': 9}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:51:42,390] Trial 80 finished with value: -0.7925692335685999 and parameters: {'l2_regularization': 0.010295688241303964, 'learning_rate': 0.024696638076076336, 'max_iter': 350, 'max_depth': 9, 'min_samples_leaf': 82, 'max_leaf_nodes': 27}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:52:20,166] Trial 81 finished with value: -0.792478518001893 and parameters: {'l2_regularization': 0.012477412275149221, 'learning_rate': 0.024966794597480243, 'max_iter': 350, 'max_depth': 13, 'min_samples_leaf': 92, 'max_leaf_nodes': 21}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:52:57,548] Trial 82 finished with value: -0.7924842868997775 and parameters: {'l2_regularization': 0.010345620022378658, 'learning_rate': 0.024872425033340195, 'max_iter': 300, 'max_depth': 15, 'min_samples_leaf': 88, 'max_leaf_nodes': 27}. Best is trial 47 with value: -0.7925694708908781.\n",
      "[I 2023-10-06 08:53:39,744] Trial 83 finished with value: -0.7926040525355023 and parameters: {'l2_regularization': 0.012037961447538227, 'learning_rate': 0.02424919113333003, 'max_iter': 350, 'max_depth': 11, 'min_samples_leaf': 80, 'max_leaf_nodes': 27}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:54:24,711] Trial 84 finished with value: -0.7924856502642846 and parameters: {'l2_regularization': 0.006610996366672632, 'learning_rate': 0.021865475252383675, 'max_iter': 350, 'max_depth': 9, 'min_samples_leaf': 82, 'max_leaf_nodes': 33}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:55:06,333] Trial 85 finished with value: -0.7924841743730436 and parameters: {'l2_regularization': 0.006606044788311778, 'learning_rate': 0.018359416941794646, 'max_iter': 300, 'max_depth': 11, 'min_samples_leaf': 80, 'max_leaf_nodes': 33}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:55:49,119] Trial 86 finished with value: -0.792503185782212 and parameters: {'l2_regularization': 0.01055419296659124, 'learning_rate': 0.0198650562992744, 'max_iter': 300, 'max_depth': 9, 'min_samples_leaf': 78, 'max_leaf_nodes': 35}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:56:28,584] Trial 87 finished with value: -0.7923798840288102 and parameters: {'l2_regularization': 0.010276457072183819, 'learning_rate': 0.01722935416896755, 'max_iter': 250, 'max_depth': 9, 'min_samples_leaf': 78, 'max_leaf_nodes': 37}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:57:10,284] Trial 88 finished with value: -0.7924343230916793 and parameters: {'l2_regularization': 0.007933106472667956, 'learning_rate': 0.019973445302725823, 'max_iter': 300, 'max_depth': 9, 'min_samples_leaf': 66, 'max_leaf_nodes': 33}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:58:00,650] Trial 89 finished with value: -0.7922453620423165 and parameters: {'l2_regularization': 0.007147507736329745, 'learning_rate': 0.014129760161219468, 'max_iter': 300, 'max_depth': 19, 'min_samples_leaf': 74, 'max_leaf_nodes': 41}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:58:34,056] Trial 90 finished with value: -0.7925767183972346 and parameters: {'l2_regularization': 0.01021414491368948, 'learning_rate': 0.022487371521859374, 'max_iter': 250, 'max_depth': 9, 'min_samples_leaf': 80, 'max_leaf_nodes': 29}. Best is trial 83 with value: -0.7926040525355023.\n",
      "[I 2023-10-06 08:59:07,479] Trial 91 finished with value: -0.7926050224827546 and parameters: {'l2_regularization': 0.0101519283986064, 'learning_rate': 0.022829064460918945, 'max_iter': 250, 'max_depth': 9, 'min_samples_leaf': 80, 'max_leaf_nodes': 29}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 08:59:35,533] Trial 92 finished with value: -0.7923585814156097 and parameters: {'l2_regularization': 0.010001118580700121, 'learning_rate': 0.021322608135482025, 'max_iter': 200, 'max_depth': 7, 'min_samples_leaf': 82, 'max_leaf_nodes': 29}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:00:08,143] Trial 93 finished with value: -0.7924392514668323 and parameters: {'l2_regularization': 0.011003926522759421, 'learning_rate': 0.022879253044630895, 'max_iter': 250, 'max_depth': 15, 'min_samples_leaf': 78, 'max_leaf_nodes': 27}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:00:42,792] Trial 94 finished with value: -0.7923184791216917 and parameters: {'l2_regularization': 0.008231044306399706, 'learning_rate': 0.0162645298844174, 'max_iter': 250, 'max_depth': 13, 'min_samples_leaf': 72, 'max_leaf_nodes': 27}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:01:29,851] Trial 95 finished with value: -0.7921876809797987 and parameters: {'l2_regularization': 0.009502522634610731, 'learning_rate': 0.025439005997760507, 'max_iter': 350, 'max_depth': 23, 'min_samples_leaf': 76, 'max_leaf_nodes': 35}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:01:54,756] Trial 96 finished with value: -0.7921143723462395 and parameters: {'l2_regularization': 0.015080031209790928, 'learning_rate': 0.0195908319497054, 'max_iter': 200, 'max_depth': 5, 'min_samples_leaf': 84, 'max_leaf_nodes': 31}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:02:33,588] Trial 97 finished with value: -0.7924820381737602 and parameters: {'l2_regularization': 0.010423843822392455, 'learning_rate': 0.022707502448088085, 'max_iter': 300, 'max_depth': 9, 'min_samples_leaf': 82, 'max_leaf_nodes': 29}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:03:03,877] Trial 98 finished with value: -0.7924585475712114 and parameters: {'l2_regularization': 0.011516736380865186, 'learning_rate': 0.02766808848674375, 'max_iter': 250, 'max_depth': 15, 'min_samples_leaf': 60, 'max_leaf_nodes': 25}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:03:52,530] Trial 99 finished with value: -0.7923465479516008 and parameters: {'l2_regularization': 0.00852141693846504, 'learning_rate': 0.01842276690924709, 'max_iter': 350, 'max_depth': 19, 'min_samples_leaf': 48, 'max_leaf_nodes': 37}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:04:35,445] Trial 100 finished with value: -0.7923384257316797 and parameters: {'l2_regularization': 0.006143644182814035, 'learning_rate': 0.01314613334433721, 'max_iter': 300, 'max_depth': 7, 'min_samples_leaf': 74, 'max_leaf_nodes': 33}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:05:25,323] Trial 101 finished with value: -0.7925064910385995 and parameters: {'l2_regularization': 0.006679131014746992, 'learning_rate': 0.018292084896458994, 'max_iter': 300, 'max_depth': 11, 'min_samples_leaf': 80, 'max_leaf_nodes': 47}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:06:18,190] Trial 102 finished with value: -0.792027844663262 and parameters: {'l2_regularization': 0.007757978166735999, 'learning_rate': 0.024496076945283413, 'max_iter': 350, 'max_depth': 11, 'min_samples_leaf': 80, 'max_leaf_nodes': 49}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:07:06,416] Trial 103 finished with value: -0.7922685623929759 and parameters: {'l2_regularization': 0.014214391736922526, 'learning_rate': 0.021139206747269337, 'max_iter': 300, 'max_depth': 13, 'min_samples_leaf': 86, 'max_leaf_nodes': 45}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:07:54,411] Trial 104 finished with value: -0.7918911630172827 and parameters: {'l2_regularization': 0.005434471931450499, 'learning_rate': 0.02929552680763844, 'max_iter': 250, 'max_depth': 27, 'min_samples_leaf': 84, 'max_leaf_nodes': 59}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:08:19,839] Trial 105 finished with value: -0.7922734765844452 and parameters: {'l2_regularization': 0.009435498161254397, 'learning_rate': 0.0272769989509912, 'max_iter': 200, 'max_depth': 5, 'min_samples_leaf': 70, 'max_leaf_nodes': 39}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:09:50,160] Trial 106 finished with value: -0.7913004326526993 and parameters: {'l2_regularization': 0.011803818064379823, 'learning_rate': 0.022636657191691063, 'max_iter': 750, 'max_depth': 17, 'min_samples_leaf': 78, 'max_leaf_nodes': 35}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:11:30,596] Trial 107 finished with value: -0.7918658486179139 and parameters: {'l2_regularization': 0.007261987040498994, 'learning_rate': 0.016394346410361638, 'max_iter': 950, 'max_depth': 11, 'min_samples_leaf': 76, 'max_leaf_nodes': 27}. Best is trial 91 with value: -0.7926050224827546.\n",
      "[I 2023-10-06 09:12:14,803] Trial 108 finished with value: -0.7926389931183399 and parameters: {'l2_regularization': 0.008533714397726131, 'learning_rate': 0.018677772327982668, 'max_iter': 350, 'max_depth': 7, 'min_samples_leaf': 88, 'max_leaf_nodes': 31}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:12:59,821] Trial 109 finished with value: -0.7925620151593674 and parameters: {'l2_regularization': 0.008829251667087413, 'learning_rate': 0.019383933605195663, 'max_iter': 350, 'max_depth': 7, 'min_samples_leaf': 80, 'max_leaf_nodes': 31}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:13:39,771] Trial 110 finished with value: -0.7926028939931811 and parameters: {'l2_regularization': 0.008434553352925098, 'learning_rate': 0.01881335393716429, 'max_iter': 350, 'max_depth': 7, 'min_samples_leaf': 80, 'max_leaf_nodes': 23}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:14:20,147] Trial 111 finished with value: -0.7925030569579621 and parameters: {'l2_regularization': 0.008749073487920728, 'learning_rate': 0.018592848508042055, 'max_iter': 350, 'max_depth': 7, 'min_samples_leaf': 80, 'max_leaf_nodes': 23}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:15:00,961] Trial 112 finished with value: -0.7925749989488194 and parameters: {'l2_regularization': 0.00886414903859232, 'learning_rate': 0.019787219738445144, 'max_iter': 350, 'max_depth': 7, 'min_samples_leaf': 80, 'max_leaf_nodes': 23}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:15:42,701] Trial 113 finished with value: -0.7925670674058195 and parameters: {'l2_regularization': 0.011217455738499724, 'learning_rate': 0.01927235585985174, 'max_iter': 300, 'max_depth': 7, 'min_samples_leaf': 72, 'max_leaf_nodes': 31}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:16:27,019] Trial 114 finished with value: -0.7924616677943395 and parameters: {'l2_regularization': 0.013644333341570157, 'learning_rate': 0.016090357060528894, 'max_iter': 350, 'max_depth': 7, 'min_samples_leaf': 72, 'max_leaf_nodes': 25}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:17:00,157] Trial 115 finished with value: -0.7920213563338602 and parameters: {'l2_regularization': 0.009131405221448691, 'learning_rate': 0.014530435902915159, 'max_iter': 250, 'max_depth': 5, 'min_samples_leaf': 86, 'max_leaf_nodes': 31}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:17:46,020] Trial 116 finished with value: -0.7925633919757422 and parameters: {'l2_regularization': 0.011992088702860636, 'learning_rate': 0.01746381549191244, 'max_iter': 400, 'max_depth': 11, 'min_samples_leaf': 74, 'max_leaf_nodes': 21}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:18:31,471] Trial 117 finished with value: -0.7923213539131999 and parameters: {'l2_regularization': 0.012113519616965008, 'learning_rate': 0.012915456185535024, 'max_iter': 400, 'max_depth': 7, 'min_samples_leaf': 74, 'max_leaf_nodes': 19}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:19:14,341] Trial 118 finished with value: -0.7924981650542943 and parameters: {'l2_regularization': 0.011486398298448615, 'learning_rate': 0.020776766373571447, 'max_iter': 400, 'max_depth': 13, 'min_samples_leaf': 70, 'max_leaf_nodes': 21}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:20:02,502] Trial 119 finished with value: -0.7924250418275818 and parameters: {'l2_regularization': 0.013744482796420729, 'learning_rate': 0.015654098123964226, 'max_iter': 350, 'max_depth': 19, 'min_samples_leaf': 68, 'max_leaf_nodes': 29}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:20:40,717] Trial 120 finished with value: -0.7923860476777799 and parameters: {'l2_regularization': 0.009811615784450453, 'learning_rate': 0.01739210905686345, 'max_iter': 350, 'max_depth': 5, 'min_samples_leaf': 82, 'max_leaf_nodes': 23}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:21:18,965] Trial 121 finished with value: -0.7924789396033887 and parameters: {'l2_regularization': 0.016753020842296552, 'learning_rate': 0.018631315881801325, 'max_iter': 300, 'max_depth': 11, 'min_samples_leaf': 76, 'max_leaf_nodes': 25}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:22:00,734] Trial 122 finished with value: -0.7925371432943261 and parameters: {'l2_regularization': 0.00863800689647566, 'learning_rate': 0.023601782044482603, 'max_iter': 400, 'max_depth': 11, 'min_samples_leaf': 80, 'max_leaf_nodes': 19}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:22:41,079] Trial 123 finished with value: -0.7925766167572248 and parameters: {'l2_regularization': 0.008405603290370786, 'learning_rate': 0.02406531399279959, 'max_iter': 400, 'max_depth': 17, 'min_samples_leaf': 84, 'max_leaf_nodes': 19}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:23:20,698] Trial 124 finished with value: -0.7925291548199563 and parameters: {'l2_regularization': 0.008366541747551377, 'learning_rate': 0.024797946670942768, 'max_iter': 400, 'max_depth': 9, 'min_samples_leaf': 84, 'max_leaf_nodes': 19}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:24:02,791] Trial 125 finished with value: -0.7923551796061321 and parameters: {'l2_regularization': 0.007491556810823887, 'learning_rate': 0.015126662969095801, 'max_iter': 350, 'max_depth': 15, 'min_samples_leaf': 78, 'max_leaf_nodes': 21}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:24:52,010] Trial 126 finished with value: -0.7923290466656431 and parameters: {'l2_regularization': 0.010811906983399767, 'learning_rate': 0.02327979048454185, 'max_iter': 400, 'max_depth': 17, 'min_samples_leaf': 74, 'max_leaf_nodes': 31}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:25:31,884] Trial 127 finished with value: -0.7924694809459647 and parameters: {'l2_regularization': 0.008869086913865832, 'learning_rate': 0.020857641706026456, 'max_iter': 350, 'max_depth': 13, 'min_samples_leaf': 82, 'max_leaf_nodes': 23}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:26:15,557] Trial 128 finished with value: -0.792343774677829 and parameters: {'l2_regularization': 0.01002383765557903, 'learning_rate': 0.029212679021447877, 'max_iter': 400, 'max_depth': 7, 'min_samples_leaf': 76, 'max_leaf_nodes': 27}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:26:57,266] Trial 129 finished with value: -0.7924843247528155 and parameters: {'l2_regularization': 0.007848967487094483, 'learning_rate': 0.019887072768201893, 'max_iter': 400, 'max_depth': 11, 'min_samples_leaf': 80, 'max_leaf_nodes': 19}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:27:32,358] Trial 130 finished with value: -0.7925337061550867 and parameters: {'l2_regularization': 0.009547263728429138, 'learning_rate': 0.02675434754130977, 'max_iter': 350, 'max_depth': 5, 'min_samples_leaf': 86, 'max_leaf_nodes': 29}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:28:07,141] Trial 131 finished with value: -0.7925188748858736 and parameters: {'l2_regularization': 0.009543922198647447, 'learning_rate': 0.026396812801072325, 'max_iter': 350, 'max_depth': 5, 'min_samples_leaf': 86, 'max_leaf_nodes': 29}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:28:43,400] Trial 132 finished with value: -0.7925394202247247 and parameters: {'l2_regularization': 0.011450887889038254, 'learning_rate': 0.023224419357214105, 'max_iter': 300, 'max_depth': 9, 'min_samples_leaf': 84, 'max_leaf_nodes': 25}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:29:18,872] Trial 133 finished with value: -0.7925222774343312 and parameters: {'l2_regularization': 0.011055822485977643, 'learning_rate': 0.022968441315295327, 'max_iter': 300, 'max_depth': 33, 'min_samples_leaf': 88, 'max_leaf_nodes': 25}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:29:50,624] Trial 134 finished with value: -0.7923793585549503 and parameters: {'l2_regularization': 0.01551965930077838, 'learning_rate': 0.017066390088526905, 'max_iter': 250, 'max_depth': 9, 'min_samples_leaf': 84, 'max_leaf_nodes': 23}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:30:22,452] Trial 135 finished with value: -0.7923942403183489 and parameters: {'l2_regularization': 0.013130480670617619, 'learning_rate': 0.020715947973225866, 'max_iter': 300, 'max_depth': 13, 'min_samples_leaf': 82, 'max_leaf_nodes': 17}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:31:01,540] Trial 136 finished with value: -0.7923734240903142 and parameters: {'l2_regularization': 0.012023106576295805, 'learning_rate': 0.030144490292054075, 'max_iter': 400, 'max_depth': 7, 'min_samples_leaf': 78, 'max_leaf_nodes': 21}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:31:40,597] Trial 137 finished with value: -0.7925135504675686 and parameters: {'l2_regularization': 0.008743507105164845, 'learning_rate': 0.02397764742893065, 'max_iter': 350, 'max_depth': 9, 'min_samples_leaf': 80, 'max_leaf_nodes': 25}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:32:25,895] Trial 138 finished with value: -0.7921788121604341 and parameters: {'l2_regularization': 0.007116014671307903, 'learning_rate': 0.012047371268043422, 'max_iter': 300, 'max_depth': 11, 'min_samples_leaf': 74, 'max_leaf_nodes': 31}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:33:09,381] Trial 139 finished with value: -0.7924834327014585 and parameters: {'l2_regularization': 0.010572059287334408, 'learning_rate': 0.01913376833810888, 'max_iter': 350, 'max_depth': 15, 'min_samples_leaf': 76, 'max_leaf_nodes': 27}. Best is trial 108 with value: -0.7926389931183399.\n",
      "[I 2023-10-06 09:33:09,383] A new study created in memory with name: xgb\n",
      "[I 2023-10-06 09:33:40,045] Trial 0 finished with value: -0.7884803073574876 and parameters: {'colsample_bytree': 0.8, 'gamma': 3.0, 'learning_rate': 0.06520579959004534, 'max_depth': 45, 'min_child_weight': 1, 'n_estimators': 650, 'subsample': 0.7}. Best is trial 0 with value: -0.7884803073574876.\n",
      "[I 2023-10-06 09:34:34,149] Trial 1 finished with value: -0.791554130486309 and parameters: {'colsample_bytree': 1.0, 'gamma': 9.5, 'learning_rate': 0.003886773904068536, 'max_depth': 39, 'min_child_weight': 14, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 1 with value: -0.791554130486309.\n",
      "[I 2023-10-06 09:35:13,863] Trial 2 finished with value: -0.7913493196844955 and parameters: {'colsample_bytree': 0.8, 'gamma': 8.5, 'learning_rate': 0.004277487922040494, 'max_depth': 41, 'min_child_weight': 5, 'n_estimators': 600, 'subsample': 0.8}. Best is trial 1 with value: -0.791554130486309.\n",
      "[I 2023-10-06 09:35:25,792] Trial 3 finished with value: -0.7921093946985189 and parameters: {'colsample_bytree': 0.9, 'gamma': 8.5, 'learning_rate': 0.027206019698577937, 'max_depth': 77, 'min_child_weight': 14, 'n_estimators': 200, 'subsample': 0.8}. Best is trial 3 with value: -0.7921093946985189.\n",
      "[I 2023-10-06 09:35:45,683] Trial 4 finished with value: -0.7924682542889435 and parameters: {'colsample_bytree': 0.8, 'gamma': 6.5, 'learning_rate': 0.04132453635540737, 'max_depth': 95, 'min_child_weight': 12, 'n_estimators': 550, 'subsample': 0.7}. Best is trial 4 with value: -0.7924682542889435.\n",
      "[I 2023-10-06 09:36:58,643] Trial 5 finished with value: -0.7919815302664253 and parameters: {'colsample_bytree': 1.0, 'gamma': 5.5, 'learning_rate': 0.002692438728781092, 'max_depth': 55, 'min_child_weight': 15, 'n_estimators': 900, 'subsample': 0.7}. Best is trial 4 with value: -0.7924682542889435.\n",
      "[I 2023-10-06 09:39:31,489] Trial 6 finished with value: -0.7890363705744321 and parameters: {'colsample_bytree': 0.6, 'gamma': 1.5, 'learning_rate': 0.006677742967912533, 'max_depth': 53, 'min_child_weight': 10, 'n_estimators': 1000, 'subsample': 1.0}. Best is trial 4 with value: -0.7924682542889435.\n",
      "[I 2023-10-06 09:40:27,421] Trial 7 finished with value: -0.7922300782417603 and parameters: {'colsample_bytree': 1.0, 'gamma': 4.5, 'learning_rate': 0.005703237420084191, 'max_depth': 45, 'min_child_weight': 15, 'n_estimators': 600, 'subsample': 0.8}. Best is trial 4 with value: -0.7924682542889435.\n",
      "[I 2023-10-06 09:41:28,635] Trial 8 finished with value: -0.7900470137885452 and parameters: {'colsample_bytree': 0.7, 'gamma': 7.5, 'learning_rate': 0.0015070599359714853, 'max_depth': 29, 'min_child_weight': 1, 'n_estimators': 800, 'subsample': 1.0}. Best is trial 4 with value: -0.7924682542889435.\n",
      "[I 2023-10-06 09:41:51,790] Trial 9 finished with value: -0.7903311505924017 and parameters: {'colsample_bytree': 0.9, 'gamma': 2.5, 'learning_rate': 0.03826996953825079, 'max_depth': 91, 'min_child_weight': 12, 'n_estimators': 300, 'subsample': 1.0}. Best is trial 4 with value: -0.7924682542889435.\n",
      "[I 2023-10-06 09:42:12,595] Trial 10 finished with value: -0.7920662333477513 and parameters: {'colsample_bytree': 0.6, 'gamma': 7.0, 'learning_rate': 0.016357634280437478, 'max_depth': 5, 'min_child_weight': 7, 'n_estimators': 450, 'subsample': 0.6}. Best is trial 4 with value: -0.7924682542889435.\n",
      "[I 2023-10-06 09:42:49,152] Trial 11 finished with value: -0.7925771756123068 and parameters: {'colsample_bytree': 0.9, 'gamma': 5.0, 'learning_rate': 0.011080069086414809, 'max_depth': 71, 'min_child_weight': 11, 'n_estimators': 550, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:43:21,379] Trial 12 finished with value: -0.7925126407992262 and parameters: {'colsample_bytree': 0.9, 'gamma': 5.0, 'learning_rate': 0.012649836644379155, 'max_depth': 99, 'min_child_weight': 10, 'n_estimators': 450, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:43:58,150] Trial 13 finished with value: -0.7922574790145837 and parameters: {'colsample_bytree': 0.9, 'gamma': 4.0, 'learning_rate': 0.01201566890098142, 'max_depth': 73, 'min_child_weight': 9, 'n_estimators': 400, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:44:32,064] Trial 14 finished with value: -0.7924783206685058 and parameters: {'colsample_bytree': 0.9, 'gamma': 5.5, 'learning_rate': 0.01443744822128458, 'max_depth': 77, 'min_child_weight': 6, 'n_estimators': 700, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:45:10,164] Trial 15 finished with value: -0.7922655072925998 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.008061008191966602, 'max_depth': 69, 'min_child_weight': 11, 'n_estimators': 450, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:45:42,785] Trial 16 finished with value: -0.7917568540164448 and parameters: {'colsample_bytree': 0.9, 'gamma': 3.5, 'learning_rate': 0.019809884978419394, 'max_depth': 99, 'min_child_weight': 4, 'n_estimators': 350, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:48:08,736] Trial 17 finished with value: -0.7847642922438425 and parameters: {'colsample_bytree': 0.7, 'gamma': 1.0, 'learning_rate': 0.010145572865356023, 'max_depth': 89, 'min_child_weight': 8, 'n_estimators': 500, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:48:27,322] Trial 18 finished with value: -0.7917608917343788 and parameters: {'colsample_bytree': 1.0, 'gamma': 5.5, 'learning_rate': 0.010598981819704811, 'max_depth': 63, 'min_child_weight': 10, 'n_estimators': 200, 'subsample': 0.8}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:48:52,734] Trial 19 finished with value: -0.7872833842378847 and parameters: {'colsample_bytree': 0.9, 'gamma': 2.0, 'learning_rate': 0.0998870565141242, 'max_depth': 83, 'min_child_weight': 12, 'n_estimators': 750, 'subsample': 1.0}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:49:10,698] Trial 20 finished with value: -0.7923047702312133 and parameters: {'colsample_bytree': 0.8, 'gamma': 6.5, 'learning_rate': 0.022356754913926632, 'max_depth': 63, 'min_child_weight': 8, 'n_estimators': 300, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:49:43,545] Trial 21 finished with value: -0.7924933931609386 and parameters: {'colsample_bytree': 0.9, 'gamma': 5.0, 'learning_rate': 0.0174783762505941, 'max_depth': 81, 'min_child_weight': 6, 'n_estimators': 700, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:50:17,143] Trial 22 finished with value: -0.7923645372457279 and parameters: {'colsample_bytree': 0.9, 'gamma': 4.5, 'learning_rate': 0.015607870790608266, 'max_depth': 83, 'min_child_weight': 3, 'n_estimators': 500, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:50:54,252] Trial 23 finished with value: -0.792016731854461 and parameters: {'colsample_bytree': 1.0, 'gamma': 5.5, 'learning_rate': 0.01120744552638531, 'max_depth': 83, 'min_child_weight': 6, 'n_estimators': 650, 'subsample': 1.0}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:52:04,253] Trial 24 finished with value: -0.7923386104282149 and parameters: {'colsample_bytree': 0.9, 'gamma': 3.5, 'learning_rate': 0.00797734795962294, 'max_depth': 99, 'min_child_weight': 9, 'n_estimators': 800, 'subsample': 0.8}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:52:26,737] Trial 25 finished with value: -0.7923104618810155 and parameters: {'colsample_bytree': 0.8, 'gamma': 6.5, 'learning_rate': 0.02617470895184318, 'max_depth': 65, 'min_child_weight': 10, 'n_estimators': 550, 'subsample': 0.9}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:53:01,690] Trial 26 finished with value: -0.79257301772369 and parameters: {'colsample_bytree': 0.8, 'gamma': 5.0, 'learning_rate': 0.016125057115687375, 'max_depth': 87, 'min_child_weight': 3, 'n_estimators': 700, 'subsample': 0.8}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:53:29,982] Trial 27 finished with value: -0.7919268014775207 and parameters: {'colsample_bytree': 0.8, 'gamma': 7.5, 'learning_rate': 0.008560932839705625, 'max_depth': 89, 'min_child_weight': 3, 'n_estimators': 400, 'subsample': 0.8}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:54:14,354] Trial 28 finished with value: -0.7924919652132909 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.011630106664823324, 'max_depth': 89, 'min_child_weight': 13, 'n_estimators': 550, 'subsample': 0.7}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:55:00,640] Trial 29 finished with value: -0.7892014688343352 and parameters: {'colsample_bytree': 0.8, 'gamma': 2.5, 'learning_rate': 0.0331915744639663, 'max_depth': 73, 'min_child_weight': 1, 'n_estimators': 650, 'subsample': 0.8}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:55:27,655] Trial 30 finished with value: -0.7925221863177647 and parameters: {'colsample_bytree': 0.8, 'gamma': 6.0, 'learning_rate': 0.023315773435827435, 'max_depth': 11, 'min_child_weight': 11, 'n_estimators': 700, 'subsample': 0.8}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:55:54,558] Trial 31 finished with value: -0.7925249912439479 and parameters: {'colsample_bytree': 0.8, 'gamma': 6.0, 'learning_rate': 0.022592120003300215, 'max_depth': 7, 'min_child_weight': 11, 'n_estimators': 750, 'subsample': 0.8}. Best is trial 11 with value: -0.7925771756123068.\n",
      "[I 2023-10-06 09:56:23,977] Trial 32 finished with value: -0.7926634511530393 and parameters: {'colsample_bytree': 0.8, 'gamma': 6.0, 'learning_rate': 0.02501238450810841, 'max_depth': 5, 'min_child_weight': 11, 'n_estimators': 850, 'subsample': 0.7}. Best is trial 32 with value: -0.7926634511530393.\n",
      "[I 2023-10-06 09:56:53,190] Trial 33 finished with value: -0.7927707212358366 and parameters: {'colsample_bytree': 0.8, 'gamma': 6.0, 'learning_rate': 0.052811696679893314, 'max_depth': 17, 'min_child_weight': 11, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 33 with value: -0.7927707212358366.\n",
      "[I 2023-10-06 09:57:20,634] Trial 34 finished with value: -0.7921665320068654 and parameters: {'colsample_bytree': 0.7, 'gamma': 8.5, 'learning_rate': 0.05172489737069446, 'max_depth': 23, 'min_child_weight': 13, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 33 with value: -0.7927707212358366.\n",
      "[I 2023-10-06 09:57:48,624] Trial 35 finished with value: -0.7919467356180363 and parameters: {'colsample_bytree': 0.8, 'gamma': 10.0, 'learning_rate': 0.06223935944092458, 'max_depth': 23, 'min_child_weight': 13, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 33 with value: -0.7927707212358366.\n",
      "[I 2023-10-06 09:58:16,846] Trial 36 finished with value: -0.7923545497547458 and parameters: {'colsample_bytree': 0.8, 'gamma': 7.5, 'learning_rate': 0.03132730362617625, 'max_depth': 19, 'min_child_weight': 14, 'n_estimators': 850, 'subsample': 0.7}. Best is trial 33 with value: -0.7927707212358366.\n",
      "[I 2023-10-06 09:58:48,710] Trial 37 finished with value: -0.7925701529798428 and parameters: {'colsample_bytree': 0.7, 'gamma': 6.0, 'learning_rate': 0.044786572033806614, 'max_depth': 35, 'min_child_weight': 9, 'n_estimators': 1000, 'subsample': 0.7}. Best is trial 33 with value: -0.7927707212358366.\n",
      "[I 2023-10-06 09:59:20,557] Trial 38 finished with value: -0.7928203024625706 and parameters: {'colsample_bytree': 0.8, 'gamma': 5.0, 'learning_rate': 0.03057386441289781, 'max_depth': 13, 'min_child_weight': 12, 'n_estimators': 850, 'subsample': 0.6}. Best is trial 38 with value: -0.7928203024625706.\n",
      "[I 2023-10-06 09:59:50,322] Trial 39 finished with value: -0.7925729985133976 and parameters: {'colsample_bytree': 0.8, 'gamma': 7.0, 'learning_rate': 0.029869036778735435, 'max_depth': 13, 'min_child_weight': 14, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 38 with value: -0.7928203024625706.\n",
      "[I 2023-10-06 10:00:18,021] Trial 40 finished with value: -0.7921089422179544 and parameters: {'colsample_bytree': 0.8, 'gamma': 8.5, 'learning_rate': 0.03492200392198079, 'max_depth': 15, 'min_child_weight': 12, 'n_estimators': 850, 'subsample': 0.6}. Best is trial 38 with value: -0.7928203024625706.\n",
      "[I 2023-10-06 10:00:55,411] Trial 41 finished with value: -0.7927632524328679 and parameters: {'colsample_bytree': 0.8, 'gamma': 5.0, 'learning_rate': 0.019511421036570224, 'max_depth': 49, 'min_child_weight': 11, 'n_estimators': 950, 'subsample': 0.7}. Best is trial 38 with value: -0.7928203024625706.\n",
      "[I 2023-10-06 10:01:32,639] Trial 42 finished with value: -0.7926453134324689 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.02745348393952782, 'max_depth': 47, 'min_child_weight': 11, 'n_estimators': 950, 'subsample': 0.7}. Best is trial 38 with value: -0.7928203024625706.\n",
      "[I 2023-10-06 10:02:04,970] Trial 43 finished with value: -0.7926275064254378 and parameters: {'colsample_bytree': 0.6, 'gamma': 4.0, 'learning_rate': 0.042783520272857395, 'max_depth': 49, 'min_child_weight': 13, 'n_estimators': 950, 'subsample': 0.7}. Best is trial 38 with value: -0.7928203024625706.\n",
      "[I 2023-10-06 10:02:42,024] Trial 44 finished with value: -0.792629718710115 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.027864720105275662, 'max_depth': 35, 'min_child_weight': 12, 'n_estimators': 950, 'subsample': 0.7}. Best is trial 38 with value: -0.7928203024625706.\n",
      "[I 2023-10-06 10:03:20,307] Trial 45 finished with value: -0.7929157741212448 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.02033050623866333, 'max_depth': 57, 'min_child_weight': 10, 'n_estimators': 850, 'subsample': 0.6}. Best is trial 45 with value: -0.7929157741212448.\n",
      "[I 2023-10-06 10:03:52,886] Trial 46 finished with value: -0.7926314473743702 and parameters: {'colsample_bytree': 0.7, 'gamma': 6.0, 'learning_rate': 0.020564478136675696, 'max_depth': 59, 'min_child_weight': 10, 'n_estimators': 850, 'subsample': 0.6}. Best is trial 45 with value: -0.7929157741212448.\n",
      "[I 2023-10-06 10:04:19,296] Trial 47 finished with value: -0.7924686028065044 and parameters: {'colsample_bytree': 0.6, 'gamma': 6.5, 'learning_rate': 0.04041652195492296, 'max_depth': 41, 'min_child_weight': 9, 'n_estimators': 800, 'subsample': 0.6}. Best is trial 45 with value: -0.7929157741212448.\n",
      "[I 2023-10-06 10:05:00,648] Trial 48 finished with value: -0.7929510765427734 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.019388755872345047, 'max_depth': 9, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 48 with value: -0.7929510765427734.\n",
      "[I 2023-10-06 10:05:56,608] Trial 49 finished with value: -0.7919216372254281 and parameters: {'colsample_bytree': 0.6, 'gamma': 3.0, 'learning_rate': 0.017914043083664993, 'max_depth': 29, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 48 with value: -0.7929510765427734.\n",
      "[I 2023-10-06 10:06:43,112] Trial 50 finished with value: -0.792885726667253 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.013694349826252354, 'max_depth': 55, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 48 with value: -0.7929510765427734.\n",
      "[I 2023-10-06 10:07:30,830] Trial 51 finished with value: -0.7928705338639883 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.013693955497179736, 'max_depth': 55, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 48 with value: -0.7929510765427734.\n",
      "[I 2023-10-06 10:08:17,596] Trial 52 finished with value: -0.792869717695216 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01426480613097702, 'max_depth': 57, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 48 with value: -0.7929510765427734.\n",
      "[I 2023-10-06 10:09:02,055] Trial 53 finished with value: -0.792953030910469 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.017775999847038752, 'max_depth': 57, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:09:50,308] Trial 54 finished with value: -0.7928823055517482 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.012878207309167045, 'max_depth': 53, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:10:36,875] Trial 55 finished with value: -0.7929346495629529 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.014879912461410523, 'max_depth': 53, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:11:33,826] Trial 56 finished with value: -0.7917947704255767 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.0, 'learning_rate': 0.016966105550257318, 'max_depth': 51, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:12:24,161] Trial 57 finished with value: -0.7928756647933032 and parameters: {'colsample_bytree': 0.6, 'gamma': 4.0, 'learning_rate': 0.013409702042930004, 'max_depth': 59, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:13:26,402] Trial 58 finished with value: -0.7926222288938716 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.00936140981630859, 'max_depth': 43, 'min_child_weight': 15, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:14:14,542] Trial 59 finished with value: -0.7929150020726117 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.012111316004915423, 'max_depth': 67, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:15:09,694] Trial 60 finished with value: -0.7925113116732503 and parameters: {'colsample_bytree': 0.6, 'gamma': 5.5, 'learning_rate': 0.006388691111130781, 'max_depth': 67, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:15:58,353] Trial 61 finished with value: -0.7929206522418071 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.012968824914742483, 'max_depth': 61, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:16:43,586] Trial 62 finished with value: -0.7929133721303983 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.015463334670898679, 'max_depth': 61, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:17:32,753] Trial 63 finished with value: -0.7923697286629758 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.018585358727883097, 'max_depth': 61, 'min_child_weight': 13, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:18:14,777] Trial 64 finished with value: -0.792886872583975 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.01549652230343717, 'max_depth': 69, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:18:59,684] Trial 65 finished with value: -0.7927883558405637 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.021215968562535555, 'max_depth': 75, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:19:44,912] Trial 66 finished with value: -0.7928003038391325 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.010602691623475536, 'max_depth': 65, 'min_child_weight': 15, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:20:27,672] Trial 67 finished with value: -0.7926938710925995 and parameters: {'colsample_bytree': 0.6, 'gamma': 5.5, 'learning_rate': 0.012333486261930482, 'max_depth': 61, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:21:39,423] Trial 68 finished with value: -0.7902956247476538 and parameters: {'colsample_bytree': 0.7, 'gamma': 2.5, 'learning_rate': 0.016484217930143305, 'max_depth': 67, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:22:34,962] Trial 69 finished with value: -0.7928216176968639 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.010229423358339127, 'max_depth': 45, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:23:20,288] Trial 70 finished with value: -0.7918363506637524 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.0, 'learning_rate': 0.023819916700739036, 'max_depth': 79, 'min_child_weight': 15, 'n_estimators': 900, 'subsample': 0.7}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:24:00,585] Trial 71 finished with value: -0.7929060990013649 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.015229098285152646, 'max_depth': 71, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:24:40,597] Trial 72 finished with value: -0.7929393473802824 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.018930353968208503, 'max_depth': 71, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:25:23,866] Trial 73 finished with value: -0.7929167836304296 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01894067535407304, 'max_depth': 63, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 53 with value: -0.792953030910469.\n",
      "[I 2023-10-06 10:26:03,956] Trial 74 finished with value: -0.7929530374510214 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.019388837266521682, 'max_depth': 65, 'min_child_weight': 13, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:26:40,095] Trial 75 finished with value: -0.7928114825270973 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.5, 'learning_rate': 0.01947075810320883, 'max_depth': 51, 'min_child_weight': 13, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:27:18,678] Trial 76 finished with value: -0.7924504581677211 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.022181682836194854, 'max_depth': 57, 'min_child_weight': 12, 'n_estimators': 800, 'subsample': 0.7}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:28:00,112] Trial 77 finished with value: -0.7926701160336271 and parameters: {'colsample_bytree': 0.6, 'gamma': 4.0, 'learning_rate': 0.024000905839861073, 'max_depth': 63, 'min_child_weight': 7, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:28:38,029] Trial 78 finished with value: -0.7927970989540524 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.01786634062236505, 'max_depth': 73, 'min_child_weight': 13, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:29:11,993] Trial 79 finished with value: -0.7928679326608261 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.026912029569913933, 'max_depth': 77, 'min_child_weight': 15, 'n_estimators': 850, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:29:46,878] Trial 80 finished with value: -0.7927996478421722 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.5, 'learning_rate': 0.02050961618253564, 'max_depth': 57, 'min_child_weight': 12, 'n_estimators': 900, 'subsample': 0.7}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:30:43,301] Trial 81 finished with value: -0.7928303415550277 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.01182370133226719, 'max_depth': 67, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:31:26,516] Trial 82 finished with value: -0.7929406739693426 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.017590221288491286, 'max_depth': 65, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:32:17,025] Trial 83 finished with value: -0.7924505101139658 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.018179609450029763, 'max_depth': 63, 'min_child_weight': 13, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:32:57,682] Trial 84 finished with value: -0.7928710362448679 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.024433653250730235, 'max_depth': 55, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:33:36,657] Trial 85 finished with value: -0.792898653814862 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.021060049581165172, 'max_depth': 59, 'min_child_weight': 12, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:34:24,778] Trial 86 finished with value: -0.7928199665226103 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.016920833519581003, 'max_depth': 65, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:35:08,607] Trial 87 finished with value: -0.7929170440812834 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.015246296980489625, 'max_depth': 71, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:35:32,044] Trial 88 finished with value: -0.7925295541127433 and parameters: {'colsample_bytree': 0.6, 'gamma': 5.0, 'learning_rate': 0.014739586222894977, 'max_depth': 71, 'min_child_weight': 13, 'n_estimators': 300, 'subsample': 0.7}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:36:23,981] Trial 89 finished with value: -0.7925371880227801 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.016284741451844064, 'max_depth': 69, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:37:09,341] Trial 90 finished with value: -0.7929127626983941 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.013543444749528757, 'max_depth': 75, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:37:52,585] Trial 91 finished with value: -0.7929035437135467 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.019148470285018693, 'max_depth': 53, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:38:36,223] Trial 92 finished with value: -0.7928720022504115 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.021298651567722995, 'max_depth': 63, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:39:10,308] Trial 93 finished with value: -0.7927994174809545 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.014595417226500582, 'max_depth': 59, 'min_child_weight': 15, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:39:44,642] Trial 94 finished with value: -0.7928846232848508 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.025834520830542364, 'max_depth': 49, 'min_child_weight': 12, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:40:20,155] Trial 95 finished with value: -0.7927603574842651 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.5, 'learning_rate': 0.017857046709327286, 'max_depth': 65, 'min_child_weight': 5, 'n_estimators': 850, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:41:09,063] Trial 96 finished with value: -0.792857336477699 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.011037991709223344, 'max_depth': 71, 'min_child_weight': 13, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:41:53,170] Trial 97 finished with value: -0.7927854717013614 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.012731737342788796, 'max_depth': 81, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:42:39,670] Trial 98 finished with value: -0.7926054935755509 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.019523483986490845, 'max_depth': 57, 'min_child_weight': 8, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:43:24,335] Trial 99 finished with value: -0.7924047867660502 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.02283716498614605, 'max_depth': 61, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:43:58,889] Trial 100 finished with value: -0.7927141790842273 and parameters: {'colsample_bytree': 0.6, 'gamma': 5.5, 'learning_rate': 0.02935978461697081, 'max_depth': 69, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:44:44,225] Trial 101 finished with value: -0.7929507403768689 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.016065425002371982, 'max_depth': 75, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:45:28,126] Trial 102 finished with value: -0.7929260494342016 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.015880284046446518, 'max_depth': 75, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:46:16,500] Trial 103 finished with value: -0.7928673221349156 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.016270022044985684, 'max_depth': 85, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 74 with value: -0.7929530374510214.\n",
      "[I 2023-10-06 10:47:02,178] Trial 104 finished with value: -0.792967756594348 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.014999227813115776, 'max_depth': 75, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:47:45,893] Trial 105 finished with value: -0.7926809832388562 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.014549907321711084, 'max_depth': 79, 'min_child_weight': 2, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:48:07,669] Trial 106 finished with value: -0.7922874582918308 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.013261803517040798, 'max_depth': 75, 'min_child_weight': 14, 'n_estimators': 250, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:49:03,064] Trial 107 finished with value: -0.7927901370645329 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.011137706456357435, 'max_depth': 73, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:49:42,787] Trial 108 finished with value: -0.7918156217511153 and parameters: {'colsample_bytree': 0.7, 'gamma': 9.0, 'learning_rate': 0.00938409496976875, 'max_depth': 77, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:50:20,419] Trial 109 finished with value: -0.792419413791237 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.016789310948859038, 'max_depth': 79, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 1.0}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:51:03,267] Trial 110 finished with value: -0.7928334016725561 and parameters: {'colsample_bytree': 0.8, 'gamma': 5.0, 'learning_rate': 0.015522671119148916, 'max_depth': 73, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:51:45,545] Trial 111 finished with value: -0.7929033140015068 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.018510460251041222, 'max_depth': 67, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:52:39,221] Trial 112 finished with value: -0.7928508038234308 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.012249952416741075, 'max_depth': 71, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:53:25,025] Trial 113 finished with value: -0.7928912052293541 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01416003515811535, 'max_depth': 69, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:54:01,156] Trial 114 finished with value: -0.7928684648395937 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.022087205668247085, 'max_depth': 65, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:54:47,853] Trial 115 finished with value: -0.7928437886152524 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.017049588509206886, 'max_depth': 75, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:55:31,854] Trial 116 finished with value: -0.7929148485980748 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01308674483655655, 'max_depth': 77, 'min_child_weight': 14, 'n_estimators': 900, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:56:11,997] Trial 117 finished with value: -0.7928916385938245 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.019841471838453126, 'max_depth': 81, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:57:07,950] Trial 118 finished with value: -0.7919831497874681 and parameters: {'colsample_bytree': 1.0, 'gamma': 3.5, 'learning_rate': 0.015589487101461857, 'max_depth': 95, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:57:49,689] Trial 119 finished with value: -0.7928697555948219 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01776735277611463, 'max_depth': 61, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.7}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:58:30,065] Trial 120 finished with value: -0.7927282842262355 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.5, 'learning_rate': 0.014158487345423663, 'max_depth': 47, 'min_child_weight': 12, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 104 with value: -0.792967756594348.\n",
      "[I 2023-10-06 10:59:01,654] Trial 121 finished with value: -0.7929753304049347 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.02033455866594284, 'max_depth': 53, 'min_child_weight': 14, 'n_estimators': 650, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 10:59:31,444] Trial 122 finished with value: -0.7929220058728668 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.02496940378954734, 'max_depth': 55, 'min_child_weight': 14, 'n_estimators': 650, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:00:05,193] Trial 123 finished with value: -0.7928126611999651 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.021201844165457732, 'max_depth': 51, 'min_child_weight': 14, 'n_estimators': 650, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:00:38,582] Trial 124 finished with value: -0.7929117546096067 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.023776153192433296, 'max_depth': 55, 'min_child_weight': 15, 'n_estimators': 750, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:01:12,608] Trial 125 finished with value: -0.7927853119459515 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.015012485208420337, 'max_depth': 53, 'min_child_weight': 14, 'n_estimators': 650, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:01:41,704] Trial 126 finished with value: -0.7927272380689577 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.025291410454758665, 'max_depth': 37, 'min_child_weight': 15, 'n_estimators': 550, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:02:21,737] Trial 127 finished with value: -0.7928775228651255 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.017578199881444538, 'max_depth': 73, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:03:02,362] Trial 128 finished with value: -0.7928592902677271 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.011904860367246332, 'max_depth': 47, 'min_child_weight': 14, 'n_estimators': 650, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:03:28,415] Trial 129 finished with value: -0.7926981378240561 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.02685795882961418, 'max_depth': 55, 'min_child_weight': 15, 'n_estimators': 500, 'subsample': 0.6}. Best is trial 121 with value: -0.7929753304049347.\n",
      "[I 2023-10-06 11:04:11,119] Trial 130 finished with value: -0.79303577537872 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01969131451779062, 'max_depth': 9, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:04:52,906] Trial 131 finished with value: -0.792966226484358 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01970312839196679, 'max_depth': 19, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:05:21,147] Trial 132 finished with value: -0.7928611009418436 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.021873397693598237, 'max_depth': 11, 'min_child_weight': 15, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:05:50,644] Trial 133 finished with value: -0.7929410067310827 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.019612906906483027, 'max_depth': 7, 'min_child_weight': 14, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:06:21,400] Trial 134 finished with value: -0.7929281660143298 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.019593146178084008, 'max_depth': 7, 'min_child_weight': 14, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:07:04,780] Trial 135 finished with value: -0.7929763638552687 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.019894003382355925, 'max_depth': 11, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:07:37,339] Trial 136 finished with value: -0.7928670741166995 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.0, 'learning_rate': 0.019338818855735025, 'max_depth': 7, 'min_child_weight': 13, 'n_estimators': 550, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:08:11,092] Trial 137 finished with value: -0.792650909156859 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.020308110231711557, 'max_depth': 9, 'min_child_weight': 13, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:08:40,061] Trial 138 finished with value: -0.7929103967492914 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.018259282348458375, 'max_depth': 5, 'min_child_weight': 13, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:09:06,865] Trial 139 finished with value: -0.7926965751327303 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.5, 'learning_rate': 0.022626187878701, 'max_depth': 17, 'min_child_weight': 15, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:09:45,515] Trial 140 finished with value: -0.7928470702764308 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.02920678731654783, 'max_depth': 9, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:10:14,092] Trial 141 finished with value: -0.7929764096808632 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.02038371036164101, 'max_depth': 11, 'min_child_weight': 14, 'n_estimators': 550, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:10:42,979] Trial 142 finished with value: -0.7929214579098718 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.020110833311447678, 'max_depth': 13, 'min_child_weight': 14, 'n_estimators': 550, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:11:17,142] Trial 143 finished with value: -0.7924494536147537 and parameters: {'colsample_bytree': 0.7, 'gamma': 1.0, 'learning_rate': 0.02273241274847426, 'max_depth': 7, 'min_child_weight': 12, 'n_estimators': 500, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:11:42,147] Trial 144 finished with value: -0.792766216250843 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.016899109012073924, 'max_depth': 21, 'min_child_weight': 14, 'n_estimators': 350, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:12:08,374] Trial 145 finished with value: -0.792740143515428 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.018598610923489998, 'max_depth': 9, 'min_child_weight': 13, 'n_estimators': 500, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:12:38,918] Trial 146 finished with value: -0.792857961716686 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.024755761493953397, 'max_depth': 15, 'min_child_weight': 15, 'n_estimators': 600, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:13:04,942] Trial 147 finished with value: -0.7928044433629424 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.02064484071857242, 'max_depth': 5, 'min_child_weight': 13, 'n_estimators': 550, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:13:29,882] Trial 148 finished with value: -0.7928865591429382 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.02751672400340528, 'max_depth': 11, 'min_child_weight': 14, 'n_estimators': 550, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:14:16,930] Trial 149 finished with value: -0.7927938364150464 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.016523829915467622, 'max_depth': 15, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:15:18,721] Trial 150 finished with value: -0.7895821700036129 and parameters: {'colsample_bytree': 0.7, 'gamma': 1.5, 'learning_rate': 0.03180472216469784, 'max_depth': 7, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:15:54,439] Trial 151 finished with value: -0.7928543938139287 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.015923216172068745, 'max_depth': 9, 'min_child_weight': 14, 'n_estimators': 700, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:16:37,753] Trial 152 finished with value: -0.7929090951817205 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01880582894919944, 'max_depth': 13, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:17:17,927] Trial 153 finished with value: -0.7930321059033745 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.02193133347531939, 'max_depth': 11, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:17:56,149] Trial 154 finished with value: -0.7928812679142485 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.02284450630216252, 'max_depth': 11, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:18:37,905] Trial 155 finished with value: -0.7928683427780038 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.02097434654937948, 'max_depth': 17, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:19:21,706] Trial 156 finished with value: -0.7930072874743205 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01775146331553152, 'max_depth': 29, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:20:06,297] Trial 157 finished with value: -0.7929120035041156 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.017785498121020794, 'max_depth': 11, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:20:30,423] Trial 158 finished with value: -0.7928678057268134 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.021518646413165837, 'max_depth': 27, 'min_child_weight': 13, 'n_estimators': 450, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:21:09,055] Trial 159 finished with value: -0.7928518818435532 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.024762909727659148, 'max_depth': 31, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:21:50,377] Trial 160 finished with value: -0.7927764616782551 and parameters: {'colsample_bytree': 0.7, 'gamma': 5.0, 'learning_rate': 0.01694312344949172, 'max_depth': 15, 'min_child_weight': 7, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:22:30,135] Trial 161 finished with value: -0.7930204864907385 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.019809591555695984, 'max_depth': 7, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:23:11,744] Trial 162 finished with value: -0.7929146073159992 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.019553394112977955, 'max_depth': 19, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:23:53,483] Trial 163 finished with value: -0.7928513444021744 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.01452786913795614, 'max_depth': 5, 'min_child_weight': 14, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:24:35,563] Trial 164 finished with value: -0.7927944053632768 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.023087219261111555, 'max_depth': 13, 'min_child_weight': 14, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:25:17,654] Trial 165 finished with value: -0.7929107536008286 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.5, 'learning_rate': 0.017705465874395802, 'max_depth': 9, 'min_child_weight': 13, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 130 with value: -0.79303577537872.\n",
      "[I 2023-10-06 11:25:57,596] Trial 166 finished with value: -0.7930401236529288 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.02082079854793157, 'max_depth': 7, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:26:38,805] Trial 167 finished with value: -0.7930020001657049 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.021133249131389625, 'max_depth': 7, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:27:19,576] Trial 168 finished with value: -0.792869132396992 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.026950473333759318, 'max_depth': 7, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:28:01,288] Trial 169 finished with value: -0.7929159254068707 and parameters: {'colsample_bytree': 0.7, 'gamma': 4.0, 'learning_rate': 0.021247624587572927, 'max_depth': 11, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:28:42,858] Trial 170 finished with value: -0.7930009103807385 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.025160967899539453, 'max_depth': 5, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:29:23,333] Trial 171 finished with value: -0.7930231844977126 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.023224074511176494, 'max_depth': 5, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:30:03,476] Trial 172 finished with value: -0.7929355780083265 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.02407530924970697, 'max_depth': 5, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:30:46,299] Trial 173 finished with value: -0.7929410742075251 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.0, 'learning_rate': 0.029003170179693227, 'max_depth': 5, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:31:27,385] Trial 174 finished with value: -0.7925600684347786 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.0325251864309393, 'max_depth': 9, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:32:01,457] Trial 175 finished with value: -0.7929143488919833 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.02918543311822786, 'max_depth': 5, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.8}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:32:38,783] Trial 176 finished with value: -0.7928031422455277 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.5, 'learning_rate': 0.03514850685695248, 'max_depth': 5, 'min_child_weight': 15, 'n_estimators': 1000, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:33:25,693] Trial 177 finished with value: -0.7921410520543662 and parameters: {'colsample_bytree': 0.7, 'gamma': 3.0, 'learning_rate': 0.02657765932383867, 'max_depth': 9, 'min_child_weight': 15, 'n_estimators': 950, 'subsample': 0.6}. Best is trial 166 with value: -0.7930401236529288.\n",
      "[I 2023-10-06 11:33:25,694] A new study created in memory with name: cat\n",
      "[I 2023-10-06 12:44:26,174] Trial 0 finished with value: -0.7908756651984412 and parameters: {'iterations': 600, 'learning_rate': 0.004677041322845278, 'depth': 15, 'random_strength': 0.7, 'bagging_temperature': 0.7, 'border_count': 81, 'l2_leaf_reg': 8}. Best is trial 0 with value: -0.7908756651984412.\n",
      "[I 2023-10-06 12:47:40,196] Trial 1 finished with value: -0.7919253073963662 and parameters: {'iterations': 650, 'learning_rate': 0.024167467984964105, 'depth': 9, 'random_strength': 0.5, 'bagging_temperature': 0.5, 'border_count': 91, 'l2_leaf_reg': 1}. Best is trial 1 with value: -0.7919253073963662.\n",
      "[I 2023-10-06 13:31:20,808] Trial 2 finished with value: -0.7874037568474472 and parameters: {'iterations': 400, 'learning_rate': 0.002145811464982664, 'depth': 15, 'random_strength': 1.0, 'bagging_temperature': 1.0, 'border_count': 71, 'l2_leaf_reg': 8}. Best is trial 1 with value: -0.7919253073963662.\n",
      "[I 2023-10-06 13:33:11,222] Trial 3 finished with value: -0.7921407225713023 and parameters: {'iterations': 800, 'learning_rate': 0.017451885081675927, 'depth': 5, 'random_strength': 0.5, 'bagging_temperature': 0.7, 'border_count': 41, 'l2_leaf_reg': 2}. Best is trial 3 with value: -0.7921407225713023.\n",
      "[I 2023-10-06 14:01:53,987] Trial 4 finished with value: -0.78448395356942 and parameters: {'iterations': 850, 'learning_rate': 0.031028436372316125, 'depth': 13, 'random_strength': 0.6, 'bagging_temperature': 1.0, 'border_count': 61, 'l2_leaf_reg': 2}. Best is trial 3 with value: -0.7921407225713023.\n"
     ]
    }
   ],
   "source": [
    "rf_study = optuna.create_study(study_name='rf')\n",
    "rf_study.optimize(rf_search, timeout=int(2 * 60 * 60))\n",
    "rf_params = rf_study.best_params\n",
    "\n",
    "et_study = optuna.create_study(study_name='et')\n",
    "et_study.optimize(et_search, timeout=int(2 * 60 * 60))\n",
    "et_params = et_study.best_params\n",
    "\n",
    "hist_study = optuna.create_study(study_name='hist')\n",
    "hist_study.optimize(hist_search, timeout=int(2 * 60 * 60))\n",
    "hist_params = hist_study.best_params\n",
    "\n",
    "xgb_study = optuna.create_study(study_name='xgb')\n",
    "xgb_study.optimize(xgb_search, timeout=int(2 * 60 * 60))\n",
    "xgb_params = xgb_study.best_params\n",
    "\n",
    "cat_study = optuna.create_study(study_name='cat')\n",
    "cat_study.optimize(cat_search, timeout=int(2 * 60 * 60))\n",
    "cat_params = cat_study.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ae6cdba6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T14:01:54.066618Z",
     "iopub.status.busy": "2023-10-06T14:01:54.066254Z",
     "iopub.status.idle": "2023-10-06T14:32:49.845931Z",
     "shell.execute_reply": "2023-10-06T14:32:49.844944Z"
    },
    "papermill": {
     "duration": 1855.830063,
     "end_time": "2023-10-06T14:32:49.871703",
     "exception": false,
     "start_time": "2023-10-06T14:01:54.041640",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 0 ==> Average Ensemble oof ROC-AUC score is ==> 0.7913420786083346\n",
      "Fold 0 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7933417933046157\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 1 ==> Average Ensemble oof ROC-AUC score is ==> 0.8012767616816177\n",
      "Fold 1 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.8025333097599758\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 2 ==> Average Ensemble oof ROC-AUC score is ==> 0.7960881996068537\n",
      "Fold 2 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7979826052856093\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 3 ==> Average Ensemble oof ROC-AUC score is ==> 0.7914849500391773\n",
      "Fold 3 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7920428008917679\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 4 ==> Average Ensemble oof ROC-AUC score is ==> 0.788211736467819\n",
      "Fold 4 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7888806174930654\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 5 ==> Average Ensemble oof ROC-AUC score is ==> 0.7848153152209969\n",
      "Fold 5 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7862379340783834\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 6 ==> Average Ensemble oof ROC-AUC score is ==> 0.7973682267462043\n",
      "Fold 6 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7982702986849399\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 7 ==> Average Ensemble oof ROC-AUC score is ==> 0.7912936365109084\n",
      "Fold 7 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7921562555446209\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 8 ==> Average Ensemble oof ROC-AUC score is ==> 0.7920712885340823\n",
      "Fold 8 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7924832297602742\n",
      "----------------------------------------------------------\n",
      "[LightGBM] [Warning] feature_fraction is set=0.8, colsample_bytree=0.6 will be ignored. Current value: feature_fraction=0.8\n",
      "Fold 9 ==> Average Ensemble oof ROC-AUC score is ==> 0.7921704253047424\n",
      "Fold 9 ==> Hill Climbing Ensemble oof ROC-AUC score is ==> 0.7931664711426814\n"
     ]
    }
   ],
   "source": [
    "# 初始化交叉检验分数，初始化预测标签\n",
    "ens_cv_scores, ens_preds = list(), list()\n",
    "# 初始化hill交叉检验分数，初始化hill预测标签\n",
    "hill_ens_cv_scores, hill_ens_preds =  list(), list()\n",
    "\n",
    "# 重复k折交叉验证\n",
    "sk = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 1, random_state = 42)\n",
    "\n",
    "# 遍历每折数据\n",
    "for i, (train_idx, test_idx) in enumerate(sk.split(X, Y)):\n",
    "    \n",
    "    # 划分训练测试集\n",
    "    X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n",
    "    Y_train, Y_test = Y.iloc[train_idx], Y.iloc[test_idx]\n",
    "    print('----------------------------------------------------------')\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练随机森林模型\n",
    "    RF_md = RandomForestClassifier(**rf_params).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    RF_pred = RF_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    RF_score = roc_auc_score(Y_test, RF_pred)\n",
    "    # 得到测试集预测概率\n",
    "    RF_pred_test = RF_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练极端森林\n",
    "    ET_md = ExtraTreesClassifier(**et_params).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    ET_pred = ET_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    ET_score = roc_auc_score(Y_test, ET_pred)\n",
    "    # 得到测试集预测概率\n",
    "    ET_pred_test = ET_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练HistGradientBoosting\n",
    "    hist_md = HistGradientBoostingClassifier(early_stopping = False,max_bins = 255,**hist_params).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    hist_pred = hist_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    hist_score = roc_auc_score(Y_test, hist_pred)\n",
    "    # 得到测试集预测概率\n",
    "    hist_pred_test = hist_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练LGBM模型\n",
    "    LGBM_md = LGBMClassifier(objective = 'binary',\n",
    "                             n_estimators = 500,\n",
    "                             max_depth = 7,\n",
    "                             learning_rate = 0.01,\n",
    "                             num_leaves = 20,\n",
    "                             reg_alpha = 3,\n",
    "                             reg_lambda = 3,\n",
    "                             subsample = 0.7,\n",
    "                             colsample_bytree = 0.7).fit(X_train, Y_train)\n",
    "    \n",
    "    # 得到测试集预测概率\n",
    "    lgb_pred = LGBM_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    lgb_score = roc_auc_score(Y_test, lgb_pred)\n",
    "    # 得到测试集预测概率\n",
    "    lgb_pred_test = LGBM_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练XGB模型\n",
    "    XGB_md = XGBClassifier(objective = 'binary:logistic',\n",
    "                           tree_method = 'hist',\n",
    "                           **xgb_params).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    xgb_pred = XGB_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    xgb_score = roc_auc_score(Y_test, xgb_pred)\n",
    "    # 得到测试集预测概率\n",
    "    xgb_pred_test = XGB_md.predict_proba(test_cv)[:, 1]\n",
    "    #--------------------------------------------------------------------------\n",
    "    # 训练CatBoost模型\n",
    "    Cat_md = CatBoostClassifier(loss_function = 'Logloss',\n",
    "                                verbose = False, \n",
    "                                task_type = 'CPU',\n",
    "                                **cat_params).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    cat_pred = Cat_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    cat_score = roc_auc_score(Y_test, cat_pred)\n",
    "    # 得到测试集预测概率\n",
    "    cat_pred_test = Cat_md.predict_proba(test_cv)[:, 1]   \n",
    "    #--------------------------------------------------------------------------\n",
    "    # 简单组合模型\n",
    "    # 各模型验证集预测概率均值\n",
    "    ens_pred_1 = (RF_pred + ET_pred + hist_pred + lgb_pred + xgb_pred + cat_pred) / 6\n",
    "    # 各模型测试集预测概率均值\n",
    "    ens_pred_2 = (RF_pred_test + ET_pred_test + hist_pred_test + lgb_pred_test + xgb_pred_test + cat_pred_test) / 6\n",
    "    # 计算AUC分数\n",
    "    ens_score_fold = roc_auc_score(Y_test, ens_pred_1)\n",
    "    # 记录组合模型验证集预测概率均值\n",
    "    ens_cv_scores.append(ens_score_fold)\n",
    "    # 记录组合模型测试集预测概率均值\n",
    "    ens_preds.append(ens_pred_2)\n",
    "    print('Fold', i, '==> Average Ensemble oof ROC-AUC score is ==>', ens_score_fold)\n",
    "    #--------------------------------------------------------------------------\n",
    "    # 加权组合模型\n",
    "    x = pd.DataFrame({'RF': RF_pred,\n",
    "                      'ET': ET_pred, \n",
    "                      'Hist': hist_pred, \n",
    "                      'LGBM': lgb_pred,\n",
    "                      'XGB': xgb_pred,\n",
    "                      'Cat': cat_pred})\n",
    "    y = Y_test\n",
    "        \n",
    "    x_test = pd.DataFrame({'RF': RF_pred_test,\n",
    "                           'ET': ET_pred_test, \n",
    "                           'Hist': hist_pred_test, \n",
    "                           'LGBM': lgb_pred_test,\n",
    "                           'XGB': xgb_pred_test,\n",
    "                           'Cat': cat_pred_test})\n",
    "    \n",
    "    hill_results = hill_climbing(x, y, x_test)\n",
    "    \n",
    "    hill_ens_score_fold = roc_auc_score(y, hill_results[0])\n",
    "    hill_ens_cv_scores.append(hill_ens_score_fold)\n",
    "    hill_ens_preds.append(hill_results[1])\n",
    "\n",
    "    print('Fold', i, '==> Hill Climbing Ensemble oof ROC-AUC score is ==>', hill_ens_score_fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6617d3dd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T14:32:49.923617Z",
     "iopub.status.busy": "2023-10-06T14:32:49.922890Z",
     "iopub.status.idle": "2023-10-06T14:32:49.931747Z",
     "shell.execute_reply": "2023-10-06T14:32:49.930904Z"
    },
    "papermill": {
     "duration": 0.037837,
     "end_time": "2023-10-06T14:32:49.933829",
     "exception": false,
     "start_time": "2023-10-06T14:32:49.895992",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # 初始化交叉检验分数，初始化预测标签\n",
    "# ens_cv_scores, ens_preds = list(), list()\n",
    "# # 初始化hill交叉检验分数，初始化hill预测标签\n",
    "# hill_ens_cv_scores, hill_ens_preds =  list(), list()\n",
    "\n",
    "# # 重复k折交叉验证\n",
    "# sk = RepeatedStratifiedKFold(n_splits = 10, n_repeats = 1, random_state = 42)\n",
    "\n",
    "# # 遍历每折数据\n",
    "# for i, (train_idx, test_idx) in enumerate(sk.split(X, Y)):\n",
    "    \n",
    "#     # 划分训练测试集\n",
    "#     X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n",
    "#     Y_train, Y_test = Y.iloc[train_idx], Y.iloc[test_idx]\n",
    "#     print('----------------------------------------------------------')\n",
    "#     #----------------------------------------------------------------------------\n",
    "#     # 训练随机森林模型\n",
    "#     RF_md = RandomForestClassifier(n_estimators = 500, \n",
    "#                                    max_depth = 7,\n",
    "#                                    min_samples_split = 15,\n",
    "#                                    min_samples_leaf = 10).fit(X_train, Y_train)\n",
    "#     # 得到验证集预测概率\n",
    "#     RF_pred = RF_md.predict_proba(X_test)[:, 1]\n",
    "#     # 计算AUC分数\n",
    "#     RF_score = roc_auc_score(Y_test, RF_pred)\n",
    "#     # 得到测试集预测概率\n",
    "#     RF_pred_test = RF_md.predict_proba(test_cv)[:, 1]\n",
    "#     #----------------------------------------------------------------------------\n",
    "#     # 训练极端森林\n",
    "#     ET_md = ExtraTreesClassifier(n_estimators = 500, \n",
    "#                                  max_depth = 7,\n",
    "#                                  min_samples_split = 15,\n",
    "#                                  min_samples_leaf = 10).fit(X_train, Y_train)\n",
    "#     # 得到验证集预测概率\n",
    "#     ET_pred = ET_md.predict_proba(X_test)[:, 1]\n",
    "#     # 计算AUC分数\n",
    "#     ET_score = roc_auc_score(Y_test, ET_pred)\n",
    "#     # 得到测试集预测概率\n",
    "#     ET_pred_test = ET_md.predict_proba(test_cv)[:, 1]\n",
    "#     #----------------------------------------------------------------------------\n",
    "#     # 训练HistGradientBoosting\n",
    "#     hist_md = HistGradientBoostingClassifier(l2_regularization = 0.01,\n",
    "#                                              early_stopping = False,\n",
    "#                                              learning_rate = 0.01,\n",
    "#                                              max_iter = 500,\n",
    "#                                              max_depth = 5,\n",
    "#                                              max_bins = 255,\n",
    "#                                              min_samples_leaf = 15,\n",
    "#                                              max_leaf_nodes = 10).fit(X_train, Y_train)\n",
    "#     # 得到验证集预测概率\n",
    "#     hist_pred = hist_md.predict_proba(X_test)[:, 1]\n",
    "#     # 计算AUC分数\n",
    "#     hist_score = roc_auc_score(Y_test, hist_pred)\n",
    "#     # 得到测试集预测概率\n",
    "#     hist_pred_test = hist_md.predict_proba(test_cv)[:, 1]\n",
    "#     #----------------------------------------------------------------------------\n",
    "#     # 训练LGBM模型\n",
    "#     LGBM_md = LGBMClassifier(objective = 'binary',\n",
    "#                              n_estimators = 500,\n",
    "#                              max_depth = 7,\n",
    "#                              learning_rate = 0.01,\n",
    "#                              num_leaves = 20,\n",
    "#                              reg_alpha = 3,\n",
    "#                              reg_lambda = 3,\n",
    "#                              subsample = 0.7,\n",
    "#                              colsample_bytree = 0.7).fit(X_train, Y_train)\n",
    "    \n",
    "#     # 得到测试集预测概率\n",
    "#     lgb_pred = LGBM_md.predict_proba(X_test)[:, 1]\n",
    "#     # 计算AUC分数\n",
    "#     lgb_score = roc_auc_score(Y_test, lgb_pred)\n",
    "#     # 得到测试集预测概率\n",
    "#     lgb_pred_test = LGBM_md.predict_proba(test_cv)[:, 1]\n",
    "#     #----------------------------------------------------------------------------\n",
    "#     # 训练XGB模型\n",
    "#     XGB_md = XGBClassifier(objective = 'binary:logistic',\n",
    "#                            tree_method = 'hist',\n",
    "#                            colsample_bytree = 0.7, \n",
    "#                            gamma = 2, \n",
    "#                            learning_rate = 0.01, \n",
    "#                            max_depth = 7, \n",
    "#                            min_child_weight = 10, \n",
    "#                            n_estimators = 500, \n",
    "#                            subsample = 0.7).fit(X_train, Y_train)\n",
    "#     # 得到验证集预测概率\n",
    "#     xgb_pred = XGB_md.predict_proba(X_test)[:, 1]\n",
    "#     # 计算AUC分数\n",
    "#     xgb_score = roc_auc_score(Y_test, xgb_pred)\n",
    "#     # 得到测试集预测概率\n",
    "#     xgb_pred_test = XGB_md.predict_proba(test_cv)[:, 1]\n",
    "#     #--------------------------------------------------------------------------\n",
    "#     # 训练CatBoost模型\n",
    "#     Cat_md = CatBoostClassifier(loss_function = 'Logloss',\n",
    "#                                 iterations = 500,\n",
    "#                                 learning_rate = 0.01,\n",
    "#                                 depth = 7,\n",
    "#                                 random_strength = 0.5,\n",
    "#                                 bagging_temperature = 0.7,\n",
    "#                                 border_count = 30,\n",
    "#                                 l2_leaf_reg = 5,\n",
    "#                                 verbose = False, \n",
    "#                                 task_type = 'CPU').fit(X_train, Y_train)\n",
    "#     # 得到验证集预测概率\n",
    "#     cat_pred = Cat_md.predict_proba(X_test)[:, 1]\n",
    "#     # 计算AUC分数\n",
    "#     cat_score = roc_auc_score(Y_test, cat_pred)\n",
    "#     # 得到测试集预测概率\n",
    "#     cat_pred_test = Cat_md.predict_proba(test_cv)[:, 1]   \n",
    "#     #--------------------------------------------------------------------------\n",
    "#     # 简单组合模型\n",
    "#     # 各模型验证集预测概率均值\n",
    "#     ens_pred_1 = (RF_pred + ET_pred + hist_pred + lgb_pred + xgb_pred + cat_pred) / 6\n",
    "#     # 各模型测试集预测概率均值\n",
    "#     ens_pred_2 = (RF_pred_test + ET_pred_test + hist_pred_test + lgb_pred_test + xgb_pred_test + cat_pred_test) / 6\n",
    "#     # 计算AUC分数\n",
    "#     ens_score_fold = roc_auc_score(Y_test, ens_pred_1)\n",
    "#     # 记录组合模型验证集预测概率均值\n",
    "#     ens_cv_scores.append(ens_score_fold)\n",
    "#     # 记录组合模型测试集预测概率均值\n",
    "#     ens_preds.append(ens_pred_2)\n",
    "#     print('Fold', i, '==> Average Ensemble oof ROC-AUC score is ==>', ens_score_fold)\n",
    "#     #--------------------------------------------------------------------------\n",
    "#     # 加权组合模型\n",
    "#     x = pd.DataFrame({'RF': RF_pred,\n",
    "#                       'ET': ET_pred, \n",
    "#                       'Hist': hist_pred, \n",
    "#                       'LGBM': lgb_pred,\n",
    "#                       'XGB': xgb_pred,\n",
    "#                       'Cat': cat_pred})\n",
    "#     y = Y_test\n",
    "        \n",
    "#     x_test = pd.DataFrame({'RF': RF_pred_test,\n",
    "#                            'ET': ET_pred_test, \n",
    "#                            'Hist': hist_pred_test, \n",
    "#                            'LGBM': lgb_pred_test,\n",
    "#                            'XGB': xgb_pred_test,\n",
    "#                            'Cat': cat_pred_test})\n",
    "    \n",
    "#     hill_results = hill_climbing(x, y, x_test)\n",
    "    \n",
    "#     hill_ens_score_fold = roc_auc_score(y, hill_results[0])\n",
    "#     hill_ens_cv_scores.append(hill_ens_score_fold)\n",
    "#     hill_ens_preds.append(hill_results[1])\n",
    "\n",
    "#     print('Fold', i, '==> Hill Climbing Ensemble oof ROC-AUC score is ==>', hill_ens_score_fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b5a6fc9b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T14:32:49.985400Z",
     "iopub.status.busy": "2023-10-06T14:32:49.984694Z",
     "iopub.status.idle": "2023-10-06T14:32:49.989788Z",
     "shell.execute_reply": "2023-10-06T14:32:49.988785Z"
    },
    "papermill": {
     "duration": 0.033975,
     "end_time": "2023-10-06T14:32:49.992408",
     "exception": false,
     "start_time": "2023-10-06T14:32:49.958433",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The average ensemble oof ROC-AUC score over the 10-folds is 0.7926122618720736\n",
      "The hill climbing ensemble oof ROC-AUC score over the 10-folds is 0.7937095315945933\n"
     ]
    }
   ],
   "source": [
    "print('The average ensemble oof ROC-AUC score over the 10-folds is', np.mean(ens_cv_scores))\n",
    "print('The hill climbing ensemble oof ROC-AUC score over the 10-folds is', np.mean(hill_ens_cv_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "57b95306",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-06T14:32:50.043752Z",
     "iopub.status.busy": "2023-10-06T14:32:50.043073Z",
     "iopub.status.idle": "2023-10-06T14:33:04.048875Z",
     "shell.execute_reply": "2023-10-06T14:33:04.047929Z"
    },
    "papermill": {
     "duration": 14.0341,
     "end_time": "2023-10-06T14:33:04.051147",
     "exception": false,
     "start_time": "2023-10-06T14:32:50.017047",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ens_preds_test = pd.DataFrame(ens_preds).apply(np.mean, axis = 0)\n",
    "\n",
    "sample_submission['defects'] = ens_preds_test\n",
    "sample_submission.to_csv('Avereage_Ensemble_Baseline_submission.csv', index = False)\n",
    "\n",
    "ens_preds_test = pd.DataFrame(hill_ens_preds).apply(np.mean, axis = 0)\n",
    "\n",
    "sample_submission['defects'] = ens_preds_test\n",
    "sample_submission.to_csv('Hill_Ensemble_Baseline_submission.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6a71cdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train.drop(columns = ['defects'], axis = 1)\n",
    "X = X.apply(lambda x: np.log1p(x))\n",
    "\n",
    "test_cv = test\n",
    "test_cv = test_cv.apply(lambda x: np.log1p(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edc92f40",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化交叉检验分数，初始化预测标签\n",
    "ens_cv_scores, ens_preds = list(), list()\n",
    "# 初始化hill交叉检验分数，初始化hill预测标签\n",
    "hill_ens_cv_scores, hill_ens_preds =  list(), list()\n",
    "\n",
    "# 重复k折交叉验证\n",
    "sk = RepeatedStratifiedKFold(n_splits = 25, n_repeats = 1, random_state = 42)\n",
    "\n",
    "# 遍历每折数据\n",
    "for i, (train_idx, test_idx) in enumerate(sk.split(X, Y)):\n",
    "    \n",
    "    # 划分训练测试集\n",
    "    X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n",
    "    Y_train, Y_test = Y.iloc[train_idx], Y.iloc[test_idx]\n",
    "    print('----------------------------------------------------------')\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练随机森林模型\n",
    "    RF_md = RandomForestClassifier(n_estimators = 500, \n",
    "                                   max_depth = 7,\n",
    "                                   min_samples_split = 15,\n",
    "                                   min_samples_leaf = 10).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    RF_pred = RF_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    RF_score = roc_auc_score(Y_test, RF_pred)\n",
    "    # 得到测试集预测概率\n",
    "    RF_pred_test = RF_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练极端森林\n",
    "    ET_md = ExtraTreesClassifier(n_estimators = 500, \n",
    "                                 max_depth = 7,\n",
    "                                 min_samples_split = 15,\n",
    "                                 min_samples_leaf = 10).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    ET_pred = ET_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    ET_score = roc_auc_score(Y_test, ET_pred)\n",
    "    # 得到测试集预测概率\n",
    "    ET_pred_test = ET_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练HistGradientBoosting\n",
    "    hist_md = HistGradientBoostingClassifier(l2_regularization = 0.01,\n",
    "                                             early_stopping = False,\n",
    "                                             learning_rate = 0.01,\n",
    "                                             max_iter = 500,\n",
    "                                             max_depth = 5,\n",
    "                                             max_bins = 255,\n",
    "                                             min_samples_leaf = 15,\n",
    "                                             max_leaf_nodes = 10).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    hist_pred = hist_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    hist_score = roc_auc_score(Y_test, hist_pred)\n",
    "    # 得到测试集预测概率\n",
    "    hist_pred_test = hist_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练LGBM模型\n",
    "    LGBM_md = LGBMClassifier(objective = 'binary',\n",
    "                             n_estimators = 500,\n",
    "                             max_depth = 7,\n",
    "                             learning_rate = 0.01,\n",
    "                             num_leaves = 20,\n",
    "                             reg_alpha = 3,\n",
    "                             reg_lambda = 3,\n",
    "                             subsample = 0.7,\n",
    "                             colsample_bytree = 0.7).fit(X_train, Y_train)\n",
    "    \n",
    "    # 得到测试集预测概率\n",
    "    lgb_pred = LGBM_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    lgb_score = roc_auc_score(Y_test, lgb_pred)\n",
    "    # 得到测试集预测概率\n",
    "    lgb_pred_test = LGBM_md.predict_proba(test_cv)[:, 1]\n",
    "    #----------------------------------------------------------------------------\n",
    "    # 训练XGB模型\n",
    "    XGB_md = XGBClassifier(objective = 'binary:logistic',\n",
    "                           tree_method = 'hist',\n",
    "                           colsample_bytree = 0.7, \n",
    "                           gamma = 2, \n",
    "                           learning_rate = 0.01, \n",
    "                           max_depth = 7, \n",
    "                           min_child_weight = 10, \n",
    "                           n_estimators = 500, \n",
    "                           subsample = 0.7).fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    xgb_pred = XGB_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    xgb_score = roc_auc_score(Y_test, xgb_pred)\n",
    "    # 得到测试集预测概率\n",
    "    xgb_pred_test = XGB_md.predict_proba(test_cv)[:, 1]\n",
    "    #--------------------------------------------------------------------------\n",
    "    # 训练CatBoost模型\n",
    "    Cat_md = CatBoostClassifier(loss_function = 'Logloss',\n",
    "                                iterations = 500,\n",
    "                                learning_rate = 0.01,\n",
    "                                depth = 7,\n",
    "                                random_strength = 0.5,\n",
    "                                bagging_temperature = 0.7,\n",
    "                                border_count = 30,\n",
    "                                l2_leaf_reg = 5,\n",
    "                                verbose = False, \n",
    "                                task_type = 'CPU').fit(X_train, Y_train)\n",
    "    # 得到验证集预测概率\n",
    "    cat_pred = Cat_md.predict_proba(X_test)[:, 1]\n",
    "    # 计算AUC分数\n",
    "    cat_score = roc_auc_score(Y_test, cat_pred)\n",
    "    # 得到测试集预测概率\n",
    "    cat_pred_test = Cat_md.predict_proba(test_cv)[:, 1]   \n",
    "    #--------------------------------------------------------------------------\n",
    "    # 简单组合模型\n",
    "    # 各模型验证集预测概率均值\n",
    "    ens_pred_1 = (RF_pred + ET_pred + hist_pred + lgb_pred + xgb_pred + cat_pred) / 6\n",
    "    # 各模型测试集预测概率均值\n",
    "    ens_pred_2 = (RF_pred_test + ET_pred_test + hist_pred_test + lgb_pred_test + xgb_pred_test + cat_pred_test) / 6\n",
    "    # 计算AUC分数\n",
    "    ens_score_fold = roc_auc_score(Y_test, ens_pred_1)\n",
    "    # 记录组合模型验证集预测概率均值\n",
    "    ens_cv_scores.append(ens_score_fold)\n",
    "    # 记录组合模型测试集预测概率均值\n",
    "    ens_preds.append(ens_pred_2)\n",
    "    print('Fold', i, '==> Average Ensemble oof ROC-AUC score is ==>', ens_score_fold)\n",
    "    #--------------------------------------------------------------------------\n",
    "    # 加权组合模型\n",
    "    x = pd.DataFrame({'RF': RF_pred,\n",
    "                      'ET': ET_pred, \n",
    "                      'Hist': hist_pred, \n",
    "                      'LGBM': lgb_pred,\n",
    "                      'XGB': xgb_pred,\n",
    "                      'Cat': cat_pred})\n",
    "    y = Y_test\n",
    "        \n",
    "    x_test = pd.DataFrame({'RF': RF_pred_test,\n",
    "                           'ET': ET_pred_test, \n",
    "                           'Hist': hist_pred_test, \n",
    "                           'LGBM': lgb_pred_test,\n",
    "                           'XGB': xgb_pred_test,\n",
    "                           'Cat': cat_pred_test})\n",
    "    \n",
    "    hill_results = hill_climbing(x, y, x_test)\n",
    "    \n",
    "    hill_ens_score_fold = roc_auc_score(y, hill_results[0])\n",
    "    hill_ens_cv_scores.append(hill_ens_score_fold)\n",
    "    hill_ens_preds.append(hill_results[1])\n",
    "\n",
    "    print('Fold', i, '==> Hill Climbing Ensemble oof ROC-AUC score is ==>', hill_ens_score_fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b101ded6",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('The average ensemble oof ROC-AUC score over the 10-folds is', np.mean(ens_cv_scores))\n",
    "print('The hill climbing ensemble oof ROC-AUC score over the 10-folds is', np.mean(hill_ens_cv_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5659b3a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ens_preds_test = pd.DataFrame(ens_preds).apply(np.mean, axis = 0)\n",
    "\n",
    "sample_submission['defects'] = ens_preds_test\n",
    "sample_submission.to_csv('Avereage_Ensemble_Baseline_submission_25_folds.csv', index = False)\n",
    "\n",
    "ens_preds_test = pd.DataFrame(hill_ens_preds).apply(np.mean, axis = 0)\n",
    "\n",
    "sample_submission['defects'] = ens_preds_test\n",
    "sample_submission.to_csv('Hill_Ensemble_Baseline_submission_25_folds.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 39913.710546,
   "end_time": "2023-10-06T14:33:05.349946",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-10-06T03:27:51.639400",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
